mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 20:19:14 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 965480ca65 |
@@ -1,2 +0,0 @@
|
||||
sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
|
||||
sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193
|
||||
sha256-linux=7c892afa4b9d1591b46bd79c976b647109a277284fddb3b98edced4b0297eda2
|
||||
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
|
||||
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=5db88c6fec94d4f269c7d9cfc128bd2adc27b3d7021127e2fa0b1daccc5f900f
|
||||
sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
|
||||
|
||||
@@ -40,21 +40,6 @@
|
||||
"invariant": "corrupt-part-arrays",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-s3",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-azure",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-gcs",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
|
||||
@@ -31,7 +31,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
./scripts/test_exact_1mib_handoff_abba.sh
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# Committed floor for the number of tests selected by the migration-critical
|
||||
# CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12).
|
||||
#
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests, with the
|
||||
# test-util feature enabled, matching the gate filter (name substrings:
|
||||
# data_movement, rebalance, decommission, source_cleanup, delete_marker) at
|
||||
# the time this file was last updated.
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests matching the
|
||||
# gate filter (name substrings: data_movement, rebalance, decommission,
|
||||
# source_cleanup, delete_marker) at the time this file was last updated.
|
||||
# CI fails if the selected count drops below this number, so renames or
|
||||
# removals that thin the gate must update this file in the same PR.
|
||||
# Adding tests does not require a bump, but bumping keeps the guard tight.
|
||||
946
|
||||
571
|
||||
|
||||
@@ -183,13 +183,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
# 4-node 4-drive distributed Actions suite: each case starts four rustfs
|
||||
# processes and up to sixteen data directories. Serialize across nextest's
|
||||
# process boundary so several 4x4 clusters never overlap.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
|
||||
# does not cross nextest process boundaries, so keep every Vault-backed test in
|
||||
# one group.
|
||||
@@ -533,27 +526,6 @@ path = "junit.xml"
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
# Serialized via e2e-cluster-nightly with no retries.
|
||||
[profile.e2e-distributed]
|
||||
default-filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
fail-fast = false
|
||||
# Decommission / rebalance cases poll for up to 180s with little stdout.
|
||||
slow-timeout = { period = "120s", terminate-after = 6 }
|
||||
|
||||
[profile.e2e-distributed.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.e2e-distributed.overrides]]
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -614,10 +586,6 @@ path = "junit.xml"
|
||||
# cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
|
||||
# excluded here because backlog#2213 promotes core heal rebuild coverage to
|
||||
# this merge/main lane while retaining nightly coverage.
|
||||
# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
|
||||
# replication, quota, observability, expand/decommission/rebalance, site
|
||||
# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# * on_demand_migration::interop_test — the ODM-20 provider interoperability
|
||||
# cases, which are meaningless without a source: they run in the dedicated
|
||||
# [profile.e2e-odm-interop] lane below, where the workflow points them at a
|
||||
@@ -639,7 +607,6 @@ default-filter = """
|
||||
package(e2e_test)
|
||||
& !test(/^protocols::/)
|
||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
& !test(/^distributed::/)
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^replication_target_matrix_test::/)
|
||||
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart.json",
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,6 @@
|
||||
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"max_age_hours": 36,
|
||||
"never_ran_grace_until": "2026-09-18T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
|
||||
@@ -582,59 +582,13 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: |
|
||||
python3 - <<'PYBUILD'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(["git", *args], text=True).strip()
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with pathlib.Path(path).open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"]
|
||||
commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
|
||||
clean_before = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_before:
|
||||
raise SystemExit("hooks binary requires a clean build checkout")
|
||||
lock_sha256 = sha256("Cargo.lock")
|
||||
lock_git_blob = git("hash-object", "Cargo.lock")
|
||||
rustc = subprocess.check_output(["rustc", "-vV"], text=True)
|
||||
host = next(line.removeprefix("host: ") for line in rustc.splitlines() if line.startswith("host: "))
|
||||
if os.environ.get("CARGO_BUILD_TARGET") or pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target")).resolve() != pathlib.Path("target").resolve():
|
||||
raise SystemExit("this artifact requires the native target/debug output")
|
||||
subprocess.run(argv, check=True)
|
||||
clean_after = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_after or commit != git("rev-parse", "HEAD") or tree != git("rev-parse", "HEAD^{tree}") or lock_sha256 != sha256("Cargo.lock"):
|
||||
raise SystemExit("hooks binary source changed while building")
|
||||
manifest = {
|
||||
"schema": 1, "commit": commit, "tree": tree,
|
||||
"clean_before": clean_before, "clean_after": clean_after,
|
||||
"lock_sha256": lock_sha256, "lock_git_blob": lock_git_blob,
|
||||
"argv": argv, "profile": "debug", "target": host,
|
||||
"features": ["e2e-test-hooks"],
|
||||
"rustc_verbose": rustc,
|
||||
"build_flags": {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_TARGET_DIR", "RUSTUP_TOOLCHAIN") if key in os.environ},
|
||||
"binary_sha256": sha256("target/debug/rustfs"),
|
||||
}
|
||||
pathlib.Path("target/debug/rustfs.e2e-startup-cas-build.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
PYBUILD
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e-startup-cas-build.json
|
||||
path: target/debug/rustfs
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -896,11 +850,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
@@ -952,36 +901,6 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Preserve startup CAS binary input
|
||||
env:
|
||||
STARTUP_CAS_INPUT: ${{ runner.temp }}/rustfs-startup-cas-input
|
||||
run: |
|
||||
python3 - <<'PYINPUT'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
source = pathlib.Path("target/debug/rustfs")
|
||||
manifest_path = source.with_name("rustfs.e2e-startup-cas-build.json")
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
target = pathlib.Path(os.environ["STARTUP_CAS_INPUT"])
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
binary = target / "rustfs"
|
||||
shutil.copy2(source, binary)
|
||||
digest = hashlib.sha256()
|
||||
with binary.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
|
||||
raise SystemExit("downloaded hooks binary identity mismatch")
|
||||
shutil.copy2(manifest_path, target / manifest_path.name)
|
||||
binary.chmod(0o755)
|
||||
PYINPUT
|
||||
|
||||
- name: Verify e2e full membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
@@ -994,10 +913,6 @@ jobs:
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
run: cargo nextest run --profile e2e-full -p e2e_test
|
||||
|
||||
- name: Upload junit
|
||||
@@ -1010,17 +925,6 @@ jobs:
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: fresh-startup-cas-evidence-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# 4-node 4-disk distributed e2e lane.
|
||||
#
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
# independent `statfs` capacity. This job runs on GitHub-hosted
|
||||
# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot
|
||||
# create filesystems: `mount -o loop` fails with ENOENT (no
|
||||
# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs
|
||||
# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason
|
||||
# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare
|
||||
# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`.
|
||||
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
|
||||
required: false
|
||||
default: ""
|
||||
schedule:
|
||||
# 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
|
||||
- cron: "53 5 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
|
||||
jobs:
|
||||
distributed:
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
# Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so
|
||||
# a shared key would mix VM and ARC pod target/ artifacts.
|
||||
cache-shared-key: ci-e2e-distributed-hosted
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Prepare isolated filesystems for pool movement
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
mkdir -p "${mount_base}"
|
||||
roots=()
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
mkdir -p "${mountpoint}"
|
||||
# Sized tmpfs reports a distinct st_dev and independent 1G
|
||||
# statfs capacity. Requires a VM runner (ubuntu-latest).
|
||||
if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then
|
||||
echo "tmpfs mount failed on $(uname -a)" >&2
|
||||
findmnt || true
|
||||
grep Cap /proc/self/status || true
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod 1777 "${mountpoint}"
|
||||
roots+=("${mountpoint}")
|
||||
done
|
||||
printf -v joined_roots '%s:' "${roots[@]}"
|
||||
echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
|
||||
|
||||
- name: Download pinned previous release
|
||||
env:
|
||||
SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$SOURCE_DIR"
|
||||
archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
|
||||
curl --fail --location --retry 3 --output "$archive" \
|
||||
"https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
|
||||
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run distributed 4-node e2e suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${FILTER}" ]; then
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
|
||||
else
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
|
||||
fi
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Unmount isolated pool filesystems
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
if mountpoint --quiet "${mountpoint}"; then
|
||||
sudo umount "${mountpoint}"
|
||||
fi
|
||||
done
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -22,7 +22,6 @@ on:
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-distributed"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
|
||||
Generated
-1
@@ -4057,7 +4057,6 @@ dependencies = [
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"suppaftp",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
|
||||
@@ -172,14 +172,6 @@ Drive timeout profile preset:
|
||||
- Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
|
||||
- Then the profile-derived default (`default` or `high_latency`).
|
||||
|
||||
## Admin peer probe timeout
|
||||
|
||||
- `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS`
|
||||
- total per-peer budget for the `server_info`/`storage_info` admin probe round; `server_info` may reconnect once and `storage_info` remains a single attempt.
|
||||
- default is `10` seconds, preserving the previous two-attempt worst-case budget.
|
||||
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
|
||||
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
|
||||
|
||||
## Startup filesystem boundary policy
|
||||
|
||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||
|
||||
@@ -39,15 +39,6 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20;
|
||||
pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
|
||||
pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30;
|
||||
|
||||
/// Total budget for one admin peer probe round, including any reconnect retry.
|
||||
///
|
||||
/// This is intentionally separate from the transport-level RPC timeout: admin
|
||||
/// probes may retry once, but the retry must consume the same round budget.
|
||||
pub const ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS: &str = "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS";
|
||||
pub const DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 10;
|
||||
pub const MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 60;
|
||||
const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS);
|
||||
|
||||
// ── Client-side internode gRPC channel tuning (P0) ──
|
||||
// These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
|
||||
// on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
|
||||
@@ -321,7 +312,6 @@ mod tests {
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20);
|
||||
assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30);
|
||||
assert_eq!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS, 10);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
|
||||
}
|
||||
|
||||
@@ -422,7 +412,6 @@ mod tests {
|
||||
"RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
|
||||
assert_eq!(ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
|
||||
assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");
|
||||
|
||||
@@ -144,6 +144,3 @@ russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -26,7 +26,6 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -172,7 +171,6 @@ the same profile for membership and execution with one nightly worker.
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
@@ -193,9 +191,6 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
cargo nextest run --profile e2e-full -p e2e_test
|
||||
# Cluster fault nightly lane
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
|
||||
# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
# Replication nightly lane; awscurl is required for STS paths
|
||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
# Fixed-port protocol nightly lane
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn emit(name: &str, value: &str) {
|
||||
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
|
||||
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
|
||||
let root = Path::new(&manifest).join("../..");
|
||||
// Cover dependency/common sources as well as this crate. HEAD/ref/index
|
||||
// changes must refresh identity even when no Rust source mtime changes.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={}", root.join(path).display());
|
||||
}
|
||||
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
|
||||
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
|
||||
git_paths.push(reference);
|
||||
}
|
||||
for path in git_paths {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
|
||||
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let mut features = std::env::vars()
|
||||
.filter_map(|(key, _)| {
|
||||
key.strip_prefix("CARGO_FEATURE_")
|
||||
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
features.sort();
|
||||
emit("COMMIT", &revision);
|
||||
emit("DIRTY", if dirty { "true" } else { "false" });
|
||||
emit("LOCK", &lock);
|
||||
emit("FEATURES", &features.join(","));
|
||||
for name in ["TARGET", "PROFILE"] {
|
||||
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
|
||||
}
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
|
||||
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
emit("RUSTFLAGS_HEX", &flags);
|
||||
}
|
||||
@@ -55,20 +55,18 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
@@ -90,15 +88,13 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sha256_hex(data: &[u8]) -> String {
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -317,8 +313,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
@@ -366,8 +360,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
@@ -421,8 +413,6 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
|
||||
@@ -1700,69 +1700,6 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a new single-node erasure pool to a stopped multi-pool cluster.
|
||||
///
|
||||
/// Used to simulate pool expansion on localhost: every pool already owns
|
||||
/// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
|
||||
/// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
|
||||
/// allocated a fresh port and empty drive directories; callers must
|
||||
/// [`Self::start`] afterwards so every process picks up the extended
|
||||
/// volumes argument. Existing data directories are left untouched.
|
||||
pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
|
||||
if self.nodes.iter().any(|node| node.process.is_some()) {
|
||||
return Err("stop the cluster before appending a pool".into());
|
||||
}
|
||||
if self.topology.drives_per_node < 2 {
|
||||
return Err(
|
||||
"append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut pools = self.topology.normalized_pools();
|
||||
for (pool_idx, nodes) in pools.iter().enumerate() {
|
||||
if nodes.len() != 1 {
|
||||
return Err(format!(
|
||||
"pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
|
||||
nodes.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
let new_idx = self.nodes.len();
|
||||
let port = RustFSTestEnvironment::find_available_port().await?;
|
||||
let address = format!("127.0.0.1:{port}");
|
||||
let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
|
||||
.map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
|
||||
.collect();
|
||||
for dir in &data_dirs {
|
||||
fs::create_dir_all(dir).await?;
|
||||
}
|
||||
|
||||
self.nodes.push(ClusterNode {
|
||||
url: format!("http://{address}"),
|
||||
address,
|
||||
data_dir: data_dirs[0].clone(),
|
||||
data_dirs,
|
||||
pool_idx: pools.len(),
|
||||
process: None,
|
||||
});
|
||||
pools.push(vec![new_idx]);
|
||||
self.topology.node_count = self.nodes.len();
|
||||
self.topology.pools = pools;
|
||||
self.node_extra_env.push(Vec::new());
|
||||
self.node_capture_log_paths.push(None);
|
||||
self.volume_proxy_addresses.push(None);
|
||||
|
||||
if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
|
||||
self.extra_env
|
||||
.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
|
||||
}
|
||||
|
||||
Ok(new_idx)
|
||||
}
|
||||
|
||||
/// Gracefully stop one cluster node and wait for its process to exit.
|
||||
///
|
||||
/// This is intentionally separate from [`Self::stop_node`]: the latter is
|
||||
|
||||
@@ -35,15 +35,11 @@ where
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..90 {
|
||||
for _ in 0..45 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.is_complete_bucket_usage_snapshot()
|
||||
&& usage.usage_snapshot_converged != Some(false)
|
||||
&& usage.buckets_usage.contains_key(bucket)
|
||||
&& predicate(&usage)
|
||||
{
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::timeout;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("killnode");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x11u8; 128 * 1024];
|
||||
put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("pwr");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x44u8; 64 * 1024];
|
||||
put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
|
||||
let bucket = unique_bucket("baddrive");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
|
||||
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
|
||||
let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(
|
||||
baseline.is_complete(),
|
||||
"replacement target did not hold a complete baseline shard: {baseline:?}"
|
||||
);
|
||||
assert!(
|
||||
!baseline.expected_part_numbers.is_empty(),
|
||||
"replacement witness must use physical part shards: {baseline:?}"
|
||||
);
|
||||
|
||||
dist.cluster.stop_node(0)?;
|
||||
let format_path = replaced_drive.join(".rustfs.sys/format.json");
|
||||
let format = std::fs::read(&format_path)?;
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
|
||||
std::fs::write(&format_path, format)?;
|
||||
let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
|
||||
|
||||
dist.cluster.start_node(0).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async {
|
||||
let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
Ok(healed.matches_manifest(&baseline))
|
||||
},
|
||||
"fresh replacement contains the original complete shard manifest",
|
||||
)
|
||||
.await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("getkill");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
|
||||
|
||||
let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
|
||||
let worker_count = 12;
|
||||
let release = Arc::new(Barrier::new(worker_count + 1));
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let client = live[idx % live.len()].clone();
|
||||
let bucket = bucket.clone();
|
||||
let body = body.clone();
|
||||
let release = release.clone();
|
||||
let started_tx = started_tx.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
|
||||
if response.content_length() != Some(body.len() as i64) {
|
||||
return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
|
||||
format!("worker {idx} received a wrong content length").into(),
|
||||
);
|
||||
}
|
||||
started_tx.send(idx)?;
|
||||
release.wait().await;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != body.as_slice() {
|
||||
return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
|
||||
}
|
||||
Ok(())
|
||||
}));
|
||||
}
|
||||
drop(started_tx);
|
||||
for _ in 0..worker_count {
|
||||
timeout(Duration::from_secs(30), started_rx.recv())
|
||||
.await?
|
||||
.ok_or("a streaming GET exited before reaching the kill barrier")?;
|
||||
}
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
release.wait().await;
|
||||
for handle in handles {
|
||||
handle.await??;
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
|
||||
let proxied_url = format!("http://{}", proxy.local_addr());
|
||||
let proxied_client = Client::from_conf(build_test_s3_config(
|
||||
&proxied_url,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
None,
|
||||
"distributed-network-chaos",
|
||||
));
|
||||
|
||||
let result: TestResult = async {
|
||||
let bucket = unique_bucket("netfault");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline = payload_for("network/baseline.bin", 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
|
||||
|
||||
proxy.set_mode(FaultMode::Blackhole);
|
||||
assert_eq!(proxy.mode(), FaultMode::Blackhole);
|
||||
if let Ok(Ok(_)) = timeout(
|
||||
Duration::from_secs(5),
|
||||
proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err("blackholed node endpoint unexpectedly completed a GET".into());
|
||||
}
|
||||
|
||||
let during = payload_for("network/during.bin", 1024 * 1024);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
|
||||
assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
proxy.shutdown().await;
|
||||
result
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let writer = &clients[idx % clients.len()];
|
||||
let reader = &clients[(idx + 1) % clients.len()];
|
||||
let copier = &clients[(idx + 2) % clients.len()];
|
||||
let mut retained = Vec::with_capacity(rounds);
|
||||
for round in 0..rounds {
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
retained.push((copy_key, body));
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
|
||||
}));
|
||||
}
|
||||
|
||||
let mut inventory = Vec::new();
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
|
||||
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("concdecom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline_client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let barrier = Arc::new(Barrier::new(17));
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..16 {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let client = &clients[idx % clients.len()];
|
||||
let key = format!("live/{idx:02}.bin");
|
||||
let body = payload_for(&key, 8 * 1024);
|
||||
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
|
||||
}));
|
||||
}
|
||||
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
barrier.wait().await;
|
||||
|
||||
let mut live_objects = Vec::new();
|
||||
for handle in handles {
|
||||
live_objects.push(handle.await??);
|
||||
}
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let checker = dist.client(2)?;
|
||||
assert_inventory(&checker, &bucket, &inventory).await?;
|
||||
for (key, body) in live_objects {
|
||||
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
|
||||
sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("integrity");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
enable_versioning(&client, &bucket).await?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
|
||||
|
||||
let versioned_key = "history/versioned.bin";
|
||||
let version_one = b"historical bytes before data movement".to_vec();
|
||||
let version_two = b"current bytes before data movement".to_vec();
|
||||
let version_one_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_one.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("historical PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version_two_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_two.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("current PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let multipart_key = "multipart/moved.bin";
|
||||
let first_part = vec![0x31; 5 * 1024 * 1024];
|
||||
let second_part = vec![0x72; 1024 * 1024];
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
|
||||
let uploaded_one = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded_two = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after_client = dist.client(2)?;
|
||||
assert_inventory(&after_client, &bucket, &inventory).await?;
|
||||
for (key, expected_hash) in before {
|
||||
let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
|
||||
}
|
||||
for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
|
||||
let got = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
|
||||
}
|
||||
let mut expected_multipart = first_part;
|
||||
expected_multipart.extend_from_slice(&second_part);
|
||||
let got_multipart = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(
|
||||
sha256_hex(got_multipart.as_ref()),
|
||||
sha256_hex(&expected_multipart),
|
||||
"multipart checksum changed after decommission"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
|
||||
put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
|
||||
wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("expand");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
|
||||
for expected_nodes in 2..=4 {
|
||||
let new_node = dist.append_pool_and_restart().await?;
|
||||
assert_eq!(new_node + 1, expected_nodes);
|
||||
assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
|
||||
}
|
||||
assert_eq!(dist.cluster.nodes.len(), 4);
|
||||
|
||||
// Prove that the expanded pool map is durable, and clear any recovery
|
||||
// latch raised while the newly-added pool replicas converged.
|
||||
dist.restart_current_binary_gracefully().await?;
|
||||
|
||||
let after_expand = dist.client(0)?;
|
||||
assert_inventory(&after_expand, &bucket, &inventory).await?;
|
||||
let peer = dist.client(3)?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("decom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
let pools_before = list_pools_json(&dist.cluster).await?;
|
||||
let pool_count = pools_before
|
||||
.as_array()
|
||||
.map(Vec::len)
|
||||
.or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
|
||||
.ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
|
||||
assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after = dist.client(2)?;
|
||||
assert_inventory(&after, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("extra");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
|
||||
let key = "multipart.bin";
|
||||
let part1 = vec![0x41u8; 5 * 1024 * 1024];
|
||||
let part2 = vec![0x42u8; 5 * 1024 * 1024];
|
||||
let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let uploaded1 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded2 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut expected = part1;
|
||||
expected.extend_from_slice(&part2);
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
|
||||
}
|
||||
|
||||
put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
|
||||
put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
|
||||
let mut seen = Vec::new();
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let listed = dist
|
||||
.client(node_idx)?
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("list/")
|
||||
.send()
|
||||
.await?;
|
||||
let keys: Vec<String> = listed
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_string))
|
||||
.collect();
|
||||
seen.push(keys);
|
||||
}
|
||||
for keys in &seen[1..] {
|
||||
assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
|
||||
}
|
||||
|
||||
let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
|
||||
assert_eq!(got, b"a");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("listed");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_idx)?;
|
||||
let name = bucket.clone();
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let name = name.clone();
|
||||
async move {
|
||||
let listed = client.list_buckets().send().await?;
|
||||
Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
|
||||
}
|
||||
},
|
||||
&format!("node {node_idx} lists {bucket}"),
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = dist.client(node_idx).expect("client");
|
||||
let name = bucket.clone();
|
||||
async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
|
||||
},
|
||||
&format!("node {node_idx} reads seed.bin"),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,35 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node 4-drive distributed e2e coverage.
|
||||
//!
|
||||
//! Selected by `[profile.e2e-distributed]` and run from
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
mod site_replication_test;
|
||||
mod upgrade_test;
|
||||
mod versioning_test;
|
||||
@@ -1,219 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use crate::object_lock::common::{
|
||||
delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::delete_object::DeleteObjectError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if code == Some("AccessDenied") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{context}: expected AccessDenied, got {error:?}").into())
|
||||
}
|
||||
}
|
||||
|
||||
async fn expect_versioned_delete_denied(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: &str,
|
||||
bypass: bool,
|
||||
context: &str,
|
||||
) -> TestResult {
|
||||
match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
|
||||
Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
|
||||
Err(error) => delete_denied(error.as_ref(), context),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let client = dist.client(0)?;
|
||||
let peer = dist.client(2)?;
|
||||
let bucket = unique_bucket("objlock");
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
compliance_key,
|
||||
b"locked-compliance",
|
||||
ObjectLockRetentionMode::Compliance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Unversioned DELETE is allowed: it only creates a delete marker. WORM
|
||||
// applies to a specific version id.
|
||||
let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
|
||||
assert_eq!(
|
||||
marker.delete_marker(),
|
||||
Some(true),
|
||||
"unversioned DELETE on a locked object must create a delete marker"
|
||||
);
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
|
||||
.await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
|
||||
|
||||
let governance_key = "governance.bin";
|
||||
let governance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
governance_key,
|
||||
b"locked-governance",
|
||||
ObjectLockRetentionMode::Governance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
|
||||
.await?;
|
||||
delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
|
||||
let deleted_governance = peer
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(governance_key)
|
||||
.version_id(&governance_version)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GOVERNANCE bypass must remove the retained version");
|
||||
assert_eq!(
|
||||
deleted_governance.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
|
||||
);
|
||||
|
||||
let hold_key = "legal-hold.bin";
|
||||
let hold_version =
|
||||
put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let bucket = unique_bucket("default-lock");
|
||||
|
||||
writer
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
|
||||
|
||||
let key = "default-governance.bin";
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"default retention payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
|
||||
|
||||
let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
|
||||
let default_retention = config
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.rule())
|
||||
.and_then(|rule| rule.default_retention())
|
||||
.ok_or("GetObjectLockConfiguration omitted default retention")?;
|
||||
assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
assert_eq!(default_retention.days(), Some(1));
|
||||
|
||||
let retention = reader
|
||||
.get_object_retention()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
|
||||
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
|
||||
.await?;
|
||||
|
||||
let plain_bucket = unique_bucket("no-lock");
|
||||
dist.create_bucket(&plain_bucket).await?;
|
||||
let configuration = ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.days(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
let error = writer
|
||||
.put_object_lock_configuration()
|
||||
.bucket(&plain_bucket)
|
||||
.object_lock_configuration(configuration)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unversioned bucket must reject Object Lock enablement");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("non-lock bucket rejection was not an S3 service error")?;
|
||||
assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
|
||||
assert_eq!(
|
||||
service_error.message(),
|
||||
Some("Object Lock configuration cannot be enabled on existing buckets"),
|
||||
"unexpected error: {error:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
|
||||
use crate::common::{admin_request, init_logging, local_http_client};
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use bytes::Bytes;
|
||||
use http::Method;
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use hyper::body::Incoming;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Request, Response};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use local_ip_address::local_ip;
|
||||
use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use serde_json::Value;
|
||||
use std::convert::Infallible;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
|
||||
let listener = TcpListener::bind("0.0.0.0:0").await?;
|
||||
let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
let method = request.method().clone();
|
||||
if let Ok(body) = request.into_body().collect().await
|
||||
&& method == Method::POST
|
||||
&& let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
|
||||
{
|
||||
if let Some(records) = payload["Records"].as_array() {
|
||||
for entry in records {
|
||||
let _ = tx.send(entry.clone());
|
||||
}
|
||||
} else {
|
||||
let _ = tx.send(payload);
|
||||
}
|
||||
}
|
||||
Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), service)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
});
|
||||
Ok((endpoint, rx, handle))
|
||||
}
|
||||
|
||||
async fn wait_for_audit_entry(
|
||||
rx: &mut mpsc::UnboundedReceiver<Value>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> TestResult<Value> {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let entry = match timeout(remaining, rx.recv()).await {
|
||||
Ok(Some(entry)) => entry,
|
||||
Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
};
|
||||
if entry["api"]["name"].as_str() == Some("s3:PutObject")
|
||||
&& entry["api"]["bucket"].as_str() == Some(bucket)
|
||||
&& entry["api"]["object"].as_str() == Some(key)
|
||||
&& entry["requestID"].as_str() == Some(request_id)
|
||||
{
|
||||
return Ok(entry);
|
||||
}
|
||||
if seen.len() < 8 {
|
||||
seen.push(format!(
|
||||
"api={:?} bucket={:?} object={:?} requestID={:?}",
|
||||
entry["api"]["name"].as_str(),
|
||||
entry["api"]["bucket"].as_str(),
|
||||
entry["api"]["object"].as_str(),
|
||||
entry["requestID"].as_str()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
|
||||
init_logging();
|
||||
let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
|
||||
let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
|
||||
let audit_env = [
|
||||
("RUSTFS_AUDIT_ENABLE", "true"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
|
||||
];
|
||||
let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
dist.cluster
|
||||
.set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
|
||||
let http = local_http_client();
|
||||
for node in &dist.cluster.nodes {
|
||||
for probe in ["ready", "live"] {
|
||||
let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
|
||||
assert!(
|
||||
response.status().is_success(),
|
||||
"node {} {probe} probe failed: {}",
|
||||
node.address,
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
|
||||
let info: Value = serde_json::from_str(&info_body)?;
|
||||
let servers = info["info"]["servers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
|
||||
assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
|
||||
|
||||
let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: Value = serde_json::from_str(&storage_body)?;
|
||||
let disks = storage["info"]["disks"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
|
||||
assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
|
||||
assert!(
|
||||
disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}),
|
||||
"storageinfo reported a drive that was not healthy and online: {storage}"
|
||||
);
|
||||
|
||||
for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
|
||||
let (status, metrics_body) = admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
|
||||
None,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
|
||||
let sample: RealtimeMetrics = serde_json::from_str(
|
||||
metrics_body
|
||||
.lines()
|
||||
.next()
|
||||
.ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
|
||||
)?;
|
||||
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
|
||||
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
|
||||
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
|
||||
}
|
||||
|
||||
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
|
||||
let targets: Value = serde_json::from_str(&targets_body)?;
|
||||
let configured = targets["audit_endpoints"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
|
||||
.iter()
|
||||
.any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
|
||||
assert!(configured, "configured audit webhook was missing: {targets}");
|
||||
|
||||
let bucket = unique_bucket("audit");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let key = "correlated/audit-object.bin";
|
||||
let put = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"distributed audit payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
|
||||
let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
|
||||
assert_eq!(
|
||||
audit["api"]["status_code"].as_i64(),
|
||||
Some(200),
|
||||
"audit entry did not report success: {audit}"
|
||||
);
|
||||
assert!(
|
||||
!audit.to_string().contains(&dist.cluster.secret_key),
|
||||
"audit entry leaked the root secret key"
|
||||
);
|
||||
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
|
||||
set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::time::Duration;
|
||||
|
||||
async fn wait_for_replication_status(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
expected: &[&str],
|
||||
timeout: Duration,
|
||||
) -> TestResult {
|
||||
wait_until(
|
||||
timeout,
|
||||
|| async {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
Ok(head
|
||||
.replication_status()
|
||||
.is_some_and(|status| expected.contains(&status.as_str())))
|
||||
},
|
||||
&format!("replication status for {bucket}/{key} in {expected:?}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
|
||||
init_logging();
|
||||
let (source, mut target) = DistCluster::start_replication_pair().await?;
|
||||
let source_bucket = unique_bucket("replsrc");
|
||||
let target_bucket = unique_bucket("repldst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
target.create_bucket(&target_bucket).await?;
|
||||
|
||||
let source_client = source.client(0)?;
|
||||
let target_client = target.client(0)?;
|
||||
enable_versioning(&source_client, &source_bucket).await?;
|
||||
enable_versioning(&target_client, &target_bucket).await?;
|
||||
|
||||
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
let key = "replicated/metadata-and-tags.bin";
|
||||
let body = b"distributed-bucket-replication".to_vec();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.metadata("origin", "four-node-source")
|
||||
.tagging("suite=distributed&shape=metadata")
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
|
||||
|
||||
let peer_read = target.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
|
||||
let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
|
||||
assert_eq!(
|
||||
replica_head
|
||||
.metadata()
|
||||
.and_then(|metadata| metadata.get("origin"))
|
||||
.map(String::as_str),
|
||||
Some("four-node-source")
|
||||
);
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
|
||||
let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
|
||||
assert_eq!(tags.get("suite"), Some(&"distributed"));
|
||||
assert_eq!(tags.get("shape"), Some(&"metadata"));
|
||||
|
||||
target.cluster.stop();
|
||||
let outage_key = "replicated/queued-during-target-outage.bin";
|
||||
let outage_body = b"retry-after-target-restart".to_vec();
|
||||
put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
|
||||
wait_for_replication_status(
|
||||
&source_client,
|
||||
&source_bucket,
|
||||
outage_key,
|
||||
&["PENDING", "FAILED"],
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
|
||||
target.cluster.start().await?;
|
||||
wait_for_ready(&target.cluster).await?;
|
||||
wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
let bucket = unique_bucket("quota");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
|
||||
|
||||
let client = dist.client(1)?;
|
||||
retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let (status, body) = super::harness::cluster_admin(
|
||||
&dist.cluster,
|
||||
Method::GET,
|
||||
&format!("/rustfs/admin/v3/quota-stats/{bucket}"),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Ok(false);
|
||||
}
|
||||
let stats: serde_json::Value =
|
||||
serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
|
||||
let usage = stats
|
||||
.get("current_usage")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
|
||||
Ok(usage >= 1024)
|
||||
},
|
||||
"quota stats observe small object",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let oversized_key = "too-big.bin";
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.body(vec![0u8; 16 * 1024].into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("hard quota must reject the oversized PUT");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("quota rejection was not an S3 service error")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(400),
|
||||
"quota rejection must be HTTP 400: {error:?}"
|
||||
);
|
||||
assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
|
||||
assert!(
|
||||
service_error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("Bucket quota exceeded")),
|
||||
"PUT must fail specifically at quota admission: {error:?}"
|
||||
);
|
||||
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an object rejected by quota must not become visible");
|
||||
assert_eq!(
|
||||
missing.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"quota-rejected object returned an unexpected HEAD result: {missing:?}"
|
||||
);
|
||||
|
||||
let listed = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
|
||||
"quota-rejected key leaked into ListObjectsV2"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::{init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3basic");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let key = "dir/object.bin";
|
||||
let body = vec![0xA5u8; 256 * 1024];
|
||||
put_object(&writer, &bucket, key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(body.len() as i64));
|
||||
assert_object_bytes(&reader, &bucket, key, &body).await?;
|
||||
|
||||
let ranged = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.range("bytes=0-15")
|
||||
.send()
|
||||
.await?;
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(ranged_body.as_ref(), &body[..16]);
|
||||
|
||||
let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, vec![key]);
|
||||
|
||||
let copy_key = "dir/object-copy.bin";
|
||||
reader
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Copy)
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
|
||||
|
||||
let moved_key = "dir/object-moved.bin";
|
||||
writer
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(moved_key)
|
||||
.copy_source(format!("{bucket}/{copy_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
|
||||
match writer.head_object().bucket(&bucket).key(copy_key).send().await {
|
||||
Ok(_) => return Err("copied source still present after rename delete".into()),
|
||||
Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
|
||||
|
||||
let presigned = writer
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
|
||||
.await?;
|
||||
let response = local_http_client().get(presigned.uri().to_string()).send().await?;
|
||||
assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
|
||||
let presigned_body = response.bytes().await?;
|
||||
assert_eq!(presigned_body.as_ref(), body.as_slice());
|
||||
|
||||
let empty_key = "empty";
|
||||
put_object(&writer, &bucket, empty_key, Vec::new()).await?;
|
||||
let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
|
||||
assert!(empty.is_empty());
|
||||
|
||||
let deleted = writer
|
||||
.delete_objects()
|
||||
.bucket(&bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(moved_key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(empty_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
|
||||
|
||||
let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3matrix");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
|
||||
let special_key = "unicode/测试 space+percent%25.txt";
|
||||
let special_body = b"metadata and tagging survive distributed routing".to_vec();
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.metadata("test-meta", "distributed")
|
||||
.tagging("purpose=compatibility&scope=four-by-four")
|
||||
.body(ByteStream::from(special_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
|
||||
assert_eq!(
|
||||
head.metadata()
|
||||
.and_then(|metadata| metadata.get("test-meta"))
|
||||
.map(String::as_str),
|
||||
Some("distributed")
|
||||
);
|
||||
assert_eq!(head.e_tag(), Some(etag.as_str()));
|
||||
let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
|
||||
let actual_tags: std::collections::BTreeMap<_, _> = tags
|
||||
.tag_set()
|
||||
.iter()
|
||||
.map(|tag| (tag.key().to_string(), tag.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
|
||||
assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
|
||||
|
||||
let conditional = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.if_match(&etag)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
|
||||
let invalid_range = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.range("bytes=999999-1000000")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unsatisfiable range must fail");
|
||||
assert_eq!(
|
||||
invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidRange"),
|
||||
"unexpected invalid-range error: {invalid_range:?}"
|
||||
);
|
||||
|
||||
let upload_key = "multipart/aborted.bin";
|
||||
let upload = writer
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
writer
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let pending = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
|
||||
writer
|
||||
.abort_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
let after_abort = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
|
||||
let aborted_head = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("aborted multipart upload must not create an object");
|
||||
assert_eq!(
|
||||
aborted_head.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
|
||||
);
|
||||
|
||||
for index in 0..113 {
|
||||
let key = format!("page/{index:04}.txt");
|
||||
put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
|
||||
}
|
||||
let mut token = None;
|
||||
let mut paged_keys = Vec::new();
|
||||
loop {
|
||||
let page = reader
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("page/")
|
||||
.max_keys(37)
|
||||
.set_continuation_token(token.take())
|
||||
.send()
|
||||
.await?;
|
||||
paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
|
||||
if page.is_truncated() != Some(true) {
|
||||
break;
|
||||
}
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.ok_or("truncated ListObjectsV2 page omitted next continuation token")?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
assert_eq!(paged_keys.len(), 113);
|
||||
let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
|
||||
assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
|
||||
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("s3move");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
let live = dist.client(2)?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed
|
||||
.contents()
|
||||
.iter()
|
||||
.any(|object| object.key() == Some("during-decommission.bin")),
|
||||
"list during decommission missed the newly written key"
|
||||
);
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&live, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let status = rebalance_status_json(&dist.cluster).await?;
|
||||
if !rebalance_running_with_progress(&status, &rebalance_id)? {
|
||||
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{init_logging, signed_request};
|
||||
use http::{Method, StatusCode};
|
||||
use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
|
||||
use std::time::Duration;
|
||||
|
||||
async fn site_replication_add(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
sites: &[PeerSite],
|
||||
) -> TestResult<ReplicateAddStatus> {
|
||||
let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
Some(serde_json::to_vec(sites)?),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("site replication add failed: {status} {body}").into());
|
||||
}
|
||||
Ok(serde_json::from_slice(&response.bytes().await?)?)
|
||||
}
|
||||
|
||||
async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
|
||||
let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let info = site_replication_info(cluster).await?;
|
||||
Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
|
||||
},
|
||||
"site replication enabled with two synchronized sites",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
let bucket = unique_bucket("siterepl");
|
||||
site_a.create_bucket(&bucket).await?;
|
||||
site_b.create_bucket(&bucket).await?;
|
||||
|
||||
let client_a = site_a.client(0)?;
|
||||
let client_b = site_b.client(0)?;
|
||||
enable_versioning(&client_a, &bucket).await?;
|
||||
enable_versioning(&client_b, &bucket).await?;
|
||||
|
||||
let sites = vec![
|
||||
PeerSite {
|
||||
name: "site-a".to_string(),
|
||||
endpoint: site_a.cluster.nodes[0].url.clone(),
|
||||
access_key: site_a.cluster.access_key.clone(),
|
||||
secret_key: site_a.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "site-b".to_string(),
|
||||
endpoint: site_b.cluster.nodes[0].url.clone(),
|
||||
access_key: site_b.cluster.access_key.clone(),
|
||||
secret_key: site_b.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"site replication add reported failure: {add_status:?}"
|
||||
);
|
||||
wait_for_site_replication_enabled(&site_a.cluster).await?;
|
||||
wait_for_site_replication_enabled(&site_b.cluster).await?;
|
||||
|
||||
let info_a = site_replication_info(&site_a.cluster).await?;
|
||||
let remote = info_a
|
||||
.sites
|
||||
.iter()
|
||||
.find(|site| site.name == "site-b")
|
||||
.ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
|
||||
assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
|
||||
let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
|
||||
assert!(
|
||||
deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
|
||||
"site peers must have two distinct non-empty deployment IDs: {info_a:?}"
|
||||
);
|
||||
assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
|
||||
assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
|
||||
|
||||
let key = "site-object.bin";
|
||||
let body = b"four-node-site-replication".to_vec();
|
||||
put_object(&client_a, &bucket, key, body.clone()).await?;
|
||||
wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
|
||||
|
||||
let peer_b = site_b.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
|
||||
|
||||
let reverse_key = "reverse/site-object.bin";
|
||||
let reverse_body = b"site-b-to-site-a".to_vec();
|
||||
put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
|
||||
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node upgrade coverage for historical objects and IAM AK/SK.
|
||||
//!
|
||||
//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
|
||||
//! mixed-version listing). This module pins the distributed contract the
|
||||
//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
|
||||
//! written on the previous release still read back, and IAM user credentials
|
||||
//! created before the upgrade still authenticate.
|
||||
//!
|
||||
//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
|
||||
//! release. The `e2e-distributed` workflow downloads that binary; a local run
|
||||
//! without it fails closed rather than skipping.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const IAM_SECRET: &str = "UpgradeTestSecretKey1";
|
||||
const WRONG_SECRET: &str = "WrongSecretKey000000";
|
||||
const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct UpgradeSeed {
|
||||
history_bucket: String,
|
||||
history_key: &'static str,
|
||||
history_body: Vec<u8>,
|
||||
versioned_bucket: String,
|
||||
versioned_key: &'static str,
|
||||
version1: String,
|
||||
version1_body: Vec<u8>,
|
||||
version2: String,
|
||||
version2_body: Vec<u8>,
|
||||
iam_bucket: String,
|
||||
iam_key: &'static str,
|
||||
iam_body: Vec<u8>,
|
||||
iam_user: String,
|
||||
iam_secret: &'static str,
|
||||
}
|
||||
|
||||
fn source_binary() -> TestResult<PathBuf> {
|
||||
let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
|
||||
format!(
|
||||
"{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
|
||||
)
|
||||
})?;
|
||||
if !path.is_file() {
|
||||
return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
|
||||
let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
|
||||
return Ok(());
|
||||
};
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_idx in 0..cluster.cluster.nodes.len() {
|
||||
let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
|
||||
cluster
|
||||
.cluster
|
||||
.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn iam_rw_policy(bucket: &str) -> String {
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}"),
|
||||
format!("arn:aws:s3:::{bucket}/*")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let admin_secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let body = body.clone();
|
||||
async move {
|
||||
put_object(&client, &bucket, &key, body).await?;
|
||||
Ok(true)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let expected = expected.to_vec();
|
||||
async move {
|
||||
let got = get_object_bytes(&client, &bucket, &key).await?;
|
||||
Ok(got == expected)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
|
||||
let history_bucket = unique_bucket("upg-hist");
|
||||
let versioned_bucket = unique_bucket("upg-ver");
|
||||
let iam_bucket = unique_bucket("upg-iam");
|
||||
dist.create_bucket(&history_bucket).await?;
|
||||
dist.create_bucket(&versioned_bucket).await?;
|
||||
dist.create_bucket(&iam_bucket).await?;
|
||||
|
||||
let root = dist.client(0)?;
|
||||
enable_versioning(&root, &versioned_bucket).await?;
|
||||
|
||||
let history_key = "plain-history.bin";
|
||||
let history_body = b"written by the previous 4-node release".to_vec();
|
||||
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
|
||||
|
||||
let versioned_key = "versioned-history.txt";
|
||||
let version1_body = b"version-one-before-upgrade".to_vec();
|
||||
let version1 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("first versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version2_body = b"version-two-before-upgrade".to_vec();
|
||||
let version2 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("second versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
|
||||
|
||||
let iam_key = "iam-history.bin";
|
||||
let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
|
||||
let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
|
||||
wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
|
||||
|
||||
Ok(UpgradeSeed {
|
||||
history_bucket,
|
||||
history_key,
|
||||
history_body,
|
||||
versioned_bucket,
|
||||
versioned_key,
|
||||
version1,
|
||||
version1_body,
|
||||
version2,
|
||||
version2_body,
|
||||
iam_bucket,
|
||||
iam_key,
|
||||
iam_body,
|
||||
iam_user,
|
||||
iam_secret: IAM_SECRET,
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
|
||||
let root_a = dist.client(0)?;
|
||||
let root_b = dist.client(3)?;
|
||||
wait_for_bytes(
|
||||
&root_b,
|
||||
&seed.history_bucket,
|
||||
seed.history_key,
|
||||
&seed.history_body,
|
||||
&format!("{context}: root GET historical object"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
|
||||
|
||||
let v1 = root_b
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version1)
|
||||
.send()
|
||||
.await?;
|
||||
let v1_body = v1.body.collect().await?.into_bytes();
|
||||
if v1_body.as_ref() != seed.version1_body.as_slice() {
|
||||
return Err(format!("{context}: version 1 bytes changed after upgrade").into());
|
||||
}
|
||||
let v2 = root_a
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version2)
|
||||
.send()
|
||||
.await?;
|
||||
let v2_body = v2.body.collect().await?.into_bytes();
|
||||
if v2_body.as_ref() != seed.version2_body.as_slice() {
|
||||
return Err(format!("{context}: version 2 bytes changed after upgrade").into());
|
||||
}
|
||||
|
||||
let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
|
||||
if !users.contains(&seed.iam_user) {
|
||||
return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
|
||||
}
|
||||
|
||||
let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
|
||||
let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
|
||||
wait_for_bytes(
|
||||
&iam_on_upgraded,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 0"),
|
||||
)
|
||||
.await?;
|
||||
wait_for_bytes(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 3"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let post_key = format!("after-upgrade-{context}.txt");
|
||||
let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
|
||||
wait_for_put(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
&post_key,
|
||||
post_body.clone(),
|
||||
&format!("{context}: IAM PUT after upgrade"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
|
||||
|
||||
let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
|
||||
match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
|
||||
Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
|
||||
Err(error) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
let rejected = code == Some("SignatureDoesNotMatch")
|
||||
|| code == Some("InvalidAccessKeyId")
|
||||
|| code == Some("AccessDenied")
|
||||
|| code == Some("InvalidArgument")
|
||||
|| error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
|
||||
if !rejected {
|
||||
return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let post_root_key = format!("root-after-{context}.bin");
|
||||
let post_root_body = format!("{context}: root write after upgrade").into_bytes();
|
||||
put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
|
||||
assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
dist.restart_with_current_binary().await?;
|
||||
assert_history_and_iam(&dist, &seed, "direct").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
|
||||
dist.replace_node_with_current_binary(0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
}
|
||||
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
|
||||
|
||||
dist.replace_node_with_current_binary(3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("version");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "versioned.txt";
|
||||
let v1_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v1".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v1 PUT omitted version ID")?
|
||||
.to_string();
|
||||
let v2_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v2".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v2 PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_versions: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
|
||||
assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
|
||||
"v1 was not the historical version: {versions:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
|
||||
"v2 was not the latest version: {versions:?}"
|
||||
);
|
||||
|
||||
let latest = get_object_bytes(&reader, &bucket, key).await?;
|
||||
assert_eq!(latest, b"v2");
|
||||
|
||||
let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let older_body = older.body.collect().await?.into_bytes();
|
||||
assert_eq!(older_body.as_ref(), b"v1");
|
||||
|
||||
let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
|
||||
let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_markers: Vec<_> = after_delete
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.filter(|marker| marker.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
matching_markers.len(),
|
||||
1,
|
||||
"delete marker missing or duplicated after current-version delete: {after_delete:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
|
||||
"DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
|
||||
);
|
||||
|
||||
let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
|
||||
match latest_after_delete {
|
||||
Ok(_) => return Err("current version should be a delete marker".into()),
|
||||
Err(error)
|
||||
if error
|
||||
.as_service_error()
|
||||
.and_then(ProvideErrorMetadata::code)
|
||||
.is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let restored_body = restored.body.collect().await?.into_bytes();
|
||||
assert_eq!(restored_body.as_ref(), b"v1");
|
||||
|
||||
writer
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(marker_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("suspend");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "suspended.txt";
|
||||
let original = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"enabled-history".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("enabled PUT omitted version ID")?
|
||||
.to_string();
|
||||
writer
|
||||
.put_bucket_versioning()
|
||||
.bucket(&bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
|
||||
put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
|
||||
let null_version_count = matching
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
matches!(
|
||||
version.version_id(),
|
||||
None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
|
||||
)
|
||||
})
|
||||
.count();
|
||||
assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
|
||||
|
||||
let historical = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(&original)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -451,40 +451,10 @@ impl JournaledHeaders {
|
||||
struct ControlState {
|
||||
scripts: HashMap<Operation, VecDeque<FaultAction>>,
|
||||
keyed_scripts: HashMap<(Operation, String), VecDeque<FaultAction>>,
|
||||
held_get: Option<HeldGetObject>,
|
||||
requests: VecDeque<RequestRecord>,
|
||||
next_sequence: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HeldGetObject {
|
||||
bucket: String,
|
||||
key: String,
|
||||
entered: watch::Sender<usize>,
|
||||
released: watch::Receiver<bool>,
|
||||
}
|
||||
|
||||
/// Holds every GET of one object, including retries, until this guard is dropped.
|
||||
#[must_use = "dropping the guard releases the held GET requests"]
|
||||
pub struct GetObjectGate {
|
||||
control: Arc<Mutex<ControlState>>,
|
||||
entered: watch::Receiver<usize>,
|
||||
released: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
impl GetObjectGate {
|
||||
pub async fn wait_until_entered(&mut self) -> Result<(), watch::error::RecvError> {
|
||||
self.entered.wait_for(|count| *count > 0).await.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GetObjectGate {
|
||||
fn drop(&mut self) {
|
||||
lock(&self.control).held_get = None;
|
||||
self.released.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
@@ -966,30 +936,6 @@ impl FakeS3Target {
|
||||
.extend(std::iter::repeat_n(action, times));
|
||||
}
|
||||
|
||||
/// Hold one exact bucket/key before any GET response can reach the client.
|
||||
/// The fixture supports one live gate; request and connection deadlines still apply.
|
||||
pub fn hold_get_object(&self, bucket: &str, key: &str) -> GetObjectGate {
|
||||
assert!(
|
||||
bucket.len() <= MAX_RETAINED_IDENTIFIER_BYTES && key.len() <= MAX_RETAINED_IDENTIFIER_BYTES,
|
||||
"held GET identifiers exceed the fixture limit"
|
||||
);
|
||||
let mut state = lock(&self.control);
|
||||
assert!(state.held_get.is_none(), "fake target already holds a GET gate");
|
||||
let (entered, entered_rx) = watch::channel(0);
|
||||
let (released, released_rx) = watch::channel(false);
|
||||
state.held_get = Some(HeldGetObject {
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
entered,
|
||||
released: released_rx,
|
||||
});
|
||||
GetObjectGate {
|
||||
control: Arc::clone(&self.control),
|
||||
entered: entered_rx,
|
||||
released,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear_faults(&self) {
|
||||
let mut state = lock(&self.control);
|
||||
state.scripts.clear();
|
||||
@@ -2297,17 +2243,6 @@ impl S3 for FakeBackend {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let held_get = lock(&self.control)
|
||||
.held_get
|
||||
.as_ref()
|
||||
.filter(|held| held.bucket == input.bucket && held.key == input.key)
|
||||
.cloned();
|
||||
if let Some(mut held) = held_get {
|
||||
held.entered.send_modify(|count| *count += 1);
|
||||
// Keep the gate installed when a request is cancelled or times out:
|
||||
// a retry must cross the same boundary before returning any bytes.
|
||||
let _ = held.released.wait_for(|released| *released).await;
|
||||
}
|
||||
let (version, versioned) = {
|
||||
let state = lock(&self.store);
|
||||
(
|
||||
@@ -2959,81 +2894,6 @@ mod tests {
|
||||
aws_sdk_s3::primitives::DateTime::from_secs(4_102_444_800)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_gate_holds_retries_and_releases_on_drop() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
let bucket = "gated-target";
|
||||
target.create_bucket(bucket);
|
||||
for key in ["held", "unrelated"] {
|
||||
target.put_seed_object(bucket, key, Bytes::from_static(b"payload"), &SeedMetadata::default());
|
||||
}
|
||||
{
|
||||
let gate = target.hold_get_object(bucket, "held");
|
||||
let request = || S3Request {
|
||||
input: GetObjectInput {
|
||||
bucket: bucket.to_string(),
|
||||
key: "held".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
method: Method::GET,
|
||||
uri: Uri::from_static("/gated-target/held"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
// Without a fault, only the gate can suspend this backend method.
|
||||
let mut first = target.backend.get_object(request());
|
||||
assert!(futures::poll!(first.as_mut()).is_pending(), "the first GET must wait at the gate");
|
||||
drop(first);
|
||||
let mut retry = target.backend.get_object(request());
|
||||
assert!(futures::poll!(retry.as_mut()).is_pending(), "a cancelled GET must not consume the gate");
|
||||
drop(gate);
|
||||
let std::task::Poll::Ready(response) = futures::poll!(retry.as_mut()) else {
|
||||
panic!("dropping the gate must release the waiting GET");
|
||||
};
|
||||
let mut body = response?.output.body.expect("released GET body");
|
||||
assert_eq!(body.next().await.transpose()?, Some(Bytes::from_static(b"payload")));
|
||||
assert!(body.next().await.is_none(), "released GET body must be complete");
|
||||
}
|
||||
let client = client(&target);
|
||||
let mut gate = target.hold_get_object(bucket, "held");
|
||||
let mut requests = tokio::task::JoinSet::new();
|
||||
let first = client.clone();
|
||||
requests.spawn(async move { get_bytes(&first, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.wait_until_entered()).await??;
|
||||
requests.abort_all();
|
||||
assert!(
|
||||
requests
|
||||
.join_next()
|
||||
.await
|
||||
.expect("first GET task")
|
||||
.expect_err("cancel the first GET attempt")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let retry = client.clone();
|
||||
requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
|
||||
timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
|
||||
drop(gate);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), requests.join_next())
|
||||
.await?
|
||||
.expect("retried GET task")??,
|
||||
Bytes::from_static(b"payload")
|
||||
);
|
||||
assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
|
||||
assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_lock_target_requires_a_checksum_on_locked_puts() -> Result<(), BoxError> {
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
|
||||
@@ -16,18 +16,15 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -37,76 +34,6 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join("background-target-restart.json").exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some((directory, run)))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
@@ -268,9 +195,8 @@ mod tests {
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
let mut node_listings = Vec::with_capacity(clients.len());
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
@@ -317,10 +243,8 @@ mod tests {
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
listed_keys.sort();
|
||||
node_listings.push(listed_keys);
|
||||
}
|
||||
Ok(node_listings)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
@@ -884,13 +808,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
@@ -938,7 +855,7 @@ mod tests {
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
@@ -1079,7 +996,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1357,7 +1274,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
cluster.start_node(interruption_node).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
@@ -1456,7 +1373,7 @@ mod tests {
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
@@ -1464,31 +1381,11 @@ mod tests {
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": expected.shard_census,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": outage_key, "version_id": null,
|
||||
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
@@ -1535,31 +1432,6 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some((directory, run)) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
|
||||
"run_id": run.run_id, "source_revision": run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.join("background-target-restart.json"))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -378,11 +378,6 @@ mod bucket_stats_regression_test;
|
||||
#[cfg(test)]
|
||||
mod distributed_startup_regression_test;
|
||||
|
||||
// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
|
||||
// quota, observability, expand/decommission/rebalance, site replication, chaos).
|
||||
#[cfg(test)]
|
||||
mod distributed;
|
||||
|
||||
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
|
||||
#[cfg(test)]
|
||||
mod tier_transition_regression_test;
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
//! local object and what the source was asked for.
|
||||
|
||||
use super::common::{
|
||||
ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with,
|
||||
AdminResponse, BoxError, OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env,
|
||||
start_configured_env_with,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
@@ -32,7 +32,7 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use local_ip_address::local_ip;
|
||||
@@ -733,242 +733,6 @@ async fn test_odm_disable_keeps_pulled_objects_and_stops_source_traffic() -> Tes
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The process switch preserves configured buckets and unfinished jobs while
|
||||
/// restoring local-only S3 behavior, including after an ordinary metadata write.
|
||||
#[tokio::test]
|
||||
async fn test_odm_global_disable_preserves_data_config_and_backfill_across_restarts() -> TestResult {
|
||||
let bucket = "odm-global-disable";
|
||||
let mut env = start_configured_env(bucket, SOURCE_BUCKET, |spec| spec.policy.list_through = true).await?;
|
||||
let pulled_key = "migrated/pulled.bin";
|
||||
let remote_key = "remote/untouched.bin";
|
||||
let pending_key = "backfill/pending.bin";
|
||||
let local_key = "local/kept.bin";
|
||||
let source_body = Bytes::from_static(b"source payload");
|
||||
let local_body = Bytes::from_static(b"client payload");
|
||||
env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new(pulled_key, source_body.clone()),
|
||||
SeedObject::new(remote_key, source_body.clone()),
|
||||
SeedObject::new(pending_key, source_body.clone()),
|
||||
],
|
||||
);
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(local_key)
|
||||
.body(local_body.clone().into())
|
||||
.send()
|
||||
.await?;
|
||||
let pulled = env.raw_get(bucket, pulled_key).await?;
|
||||
assert_eq!(pulled.status, 200);
|
||||
assert_eq!(pulled.header(ODM_RESPONSE_HEADER), Some("source"));
|
||||
assert_eq!(pulled.body, source_body);
|
||||
let stored = env.raw_get(bucket, pulled_key).await?;
|
||||
assert_eq!(stored.status, 200);
|
||||
assert_eq!(stored.header(ODM_RESPONSE_HEADER), None, "the inline pull has committed locally");
|
||||
assert_eq!(stored.body, source_body);
|
||||
let config = env.get_config(bucket).await?;
|
||||
assert_eq!(config.status, 200, "{}", config.body);
|
||||
let config = config.json()?;
|
||||
|
||||
// Hold every attempt until the process has exited, so retries cannot commit
|
||||
// the only backfill object before the crash. The start checkpoint exists.
|
||||
let mut pending_get = env.source.hold_get_object(SOURCE_BUCKET, pending_key);
|
||||
let started = env
|
||||
.start_backfill(
|
||||
bucket,
|
||||
BackfillRequest {
|
||||
prefix: Some("backfill/".to_string()),
|
||||
..BackfillRequest::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(started.status, 200, "{}", started.body);
|
||||
let job_id = started.json()?["job"]["job_id"].as_str().ok_or("missing job ID")?.to_string();
|
||||
tokio::time::timeout(Duration::from_secs(10), pending_get.wait_until_entered())
|
||||
.await
|
||||
.expect("backfill never reached the held source GET")?;
|
||||
let process = env.rustfs.process.as_mut().ok_or("missing RustFS process before crash")?;
|
||||
assert!(process.try_wait()?.is_none(), "RustFS exited before the controlled crash");
|
||||
process.kill()?;
|
||||
let stopped = process.wait()?;
|
||||
assert!(!stopped.success(), "the interrupted process must exit after being killed");
|
||||
drop(env.rustfs.process.take());
|
||||
drop(pending_get);
|
||||
env.source.take_requests();
|
||||
env.rustfs
|
||||
.restart_server_preserving_data(vec![], &[(ODM_MODULE_SWITCH_ENV, "false"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")])
|
||||
.await?;
|
||||
|
||||
let off_config = env.get_config(bucket).await?;
|
||||
assert_eq!(off_config.status, 200, "{}", off_config.body);
|
||||
assert_eq!(off_config.json()?, config, "the saved configuration and timestamp survive disabling");
|
||||
let status = env.status_json(bucket).await?;
|
||||
assert_eq!(status["configured"], true, "{status}");
|
||||
assert_eq!(status["enabled"], true, "the bucket remains configured as enabled: {status}");
|
||||
assert_eq!(status["module_enabled"], false, "{status}");
|
||||
assert_eq!(status["counters"], Value::Null, "no bucket runtime is installed: {status}");
|
||||
let checkpoint = env.backfill_job(bucket).await?.ok_or("disabled module lost the checkpoint")?;
|
||||
assert_eq!(checkpoint["job_id"], job_id);
|
||||
assert_eq!(checkpoint["state"], "running", "the interrupted job is retained: {checkpoint}");
|
||||
|
||||
for (key, body) in [(local_key, &local_body), (pulled_key, &source_body)] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 200);
|
||||
assert_eq!(&get.body, body);
|
||||
assert_eq!(get.header(ODM_RESPONSE_HEADER), None);
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
for key in [remote_key, pending_key] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 404, "disabled source GET {key}: {}", String::from_utf8_lossy(&get.body));
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await;
|
||||
let err = head.expect_err("a source-only object must remain absent locally");
|
||||
assert_eq!(err.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
|
||||
let replacement = Bytes::from_static(b"written while the module is off");
|
||||
for key in [local_key, "local/deleted.bin"] {
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(replacement.clone().into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
env.client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("local/deleted.bin")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(env.raw_get(bucket, "local/deleted.bin").await?.status, 404);
|
||||
assert_eq!(env.raw_get(bucket, local_key).await?.body, replacement);
|
||||
|
||||
// Both wire protocols must finish their local pages even though the saved
|
||||
// configuration still requests list-through.
|
||||
for use_v2 in [false, true] {
|
||||
let mut cursor = None;
|
||||
let mut listed = Vec::new();
|
||||
for page_number in 0..2 {
|
||||
let (keys, truncated, next) = if use_v2 {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(1)
|
||||
.set_continuation_token(cursor)
|
||||
.send()
|
||||
.await?;
|
||||
(
|
||||
page.contents()
|
||||
.iter()
|
||||
.map(|object| object.key().expect("listed key").to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
page.is_truncated(),
|
||||
page.next_continuation_token().map(str::to_string),
|
||||
)
|
||||
} else {
|
||||
let page = env
|
||||
.client
|
||||
.list_objects()
|
||||
.bucket(bucket)
|
||||
.max_keys(1)
|
||||
.set_marker(cursor)
|
||||
.send()
|
||||
.await?;
|
||||
// V1 may omit NextMarker without a delimiter; clients then
|
||||
// continue from the last returned key.
|
||||
let next = page.next_marker().or_else(|| {
|
||||
if page.is_truncated() == Some(true) {
|
||||
page.contents().last().and_then(|object| object.key())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
(
|
||||
page.contents()
|
||||
.iter()
|
||||
.map(|object| object.key().expect("listed key").to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
page.is_truncated(),
|
||||
next.map(str::to_string),
|
||||
)
|
||||
};
|
||||
assert_eq!(keys.len(), 1, "one local key per page, V2={use_v2}");
|
||||
assert_eq!(truncated, Some(page_number == 0), "local pagination must terminate, V2={use_v2}");
|
||||
if page_number == 0 {
|
||||
assert!(next.as_ref().is_some_and(|value| !value.is_empty()), "missing local cursor, V2={use_v2}");
|
||||
}
|
||||
cursor = next;
|
||||
listed.extend(keys);
|
||||
}
|
||||
assert_eq!(listed, [local_key, pulled_key], "source-only keys must stay absent, V2={use_v2}");
|
||||
}
|
||||
|
||||
let spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
for response in [
|
||||
env.configure_source(bucket, &spec).await?,
|
||||
env.validate_source(bucket, &spec).await?,
|
||||
env.backfill(bucket, BackfillOp::Start(BackfillRequest::default())).await?,
|
||||
] {
|
||||
assert_eq!(response.status, 400, "{}", response.body);
|
||||
assert!(response.body.contains("OnDemandMigrationDisabled"), "{}", response.body);
|
||||
}
|
||||
let tagging = Tagging::builder()
|
||||
.tag_set(Tag::builder().key("module").value("disabled").build()?)
|
||||
.build()?;
|
||||
env.client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(tagging.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(env.get_config(bucket).await?.json()?, config, "an unrelated metadata write preserves ODM");
|
||||
assert_eq!(
|
||||
env.backfill_job(bucket).await?,
|
||||
Some(checkpoint),
|
||||
"no recovery or checkpoint update while disabled"
|
||||
);
|
||||
assert!(
|
||||
env.source.requests().is_empty(),
|
||||
"disabled startup and all requests must leave the source untouched"
|
||||
);
|
||||
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
env.wait_until_source_consulted(bucket).await?;
|
||||
assert_eq!(
|
||||
env.get_config(bucket).await?.json()?,
|
||||
config,
|
||||
"reenabling uses the persisted configuration"
|
||||
);
|
||||
let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
assert_eq!(tags.tag_set(), tagging.tag_set(), "the ordinary metadata write also persists");
|
||||
let resumed = env.raw_get(bucket, remote_key).await?;
|
||||
assert_eq!(resumed.status, 200);
|
||||
assert_eq!(resumed.header(ODM_RESPONSE_HEADER), Some("source"));
|
||||
assert_eq!(resumed.body, source_body, "stored credentials still authenticate without reconfiguration");
|
||||
let completed = env
|
||||
.wait_for_backfill(bucket, SETTLE, |job| job["state"] == "completed")
|
||||
.await?;
|
||||
assert_eq!(completed["job_id"], job_id, "the interrupted job resumes without a new start");
|
||||
assert_eq!(completed["failed"], 0, "{completed}");
|
||||
for (key, body) in [
|
||||
(local_key, &replacement),
|
||||
(pulled_key, &source_body),
|
||||
(pending_key, &source_body),
|
||||
] {
|
||||
let get = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(get.status, 200);
|
||||
assert_eq!(&get.body, body);
|
||||
assert_eq!(get.header(ODM_RESPONSE_HEADER), None, "{key} remains stored locally");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Case 19: the admin surface an operator sees — the configuration read back
|
||||
/// without its secret, and a status document whose counters match the source
|
||||
/// journal exactly.
|
||||
|
||||
@@ -118,8 +118,6 @@ hotpath-cpu = [
|
||||
# injection, xl.meta transition assertions) via `api::tier::test_util`.
|
||||
# Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6).
|
||||
test-util = []
|
||||
# Observes real startup CAS only in the dedicated E2E binary.
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
|
||||
@@ -69,13 +69,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_control {
|
||||
pub use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
|
||||
inspect_recovery_control, list_recovery_controls,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
@@ -96,9 +89,8 @@ pub mod bucket {
|
||||
#[allow(clippy::module_inception)]
|
||||
pub mod lifecycle {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
object_opts_from_object_info,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
|
||||
TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -159,17 +151,15 @@ pub mod bucket {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
|
||||
delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
|
||||
get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock,
|
||||
update_under_transaction_lock_at,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
|
||||
@@ -368,8 +358,6 @@ pub mod data_usage {
|
||||
pub mod disk {
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
pub use crate::disk::local::ScanGuard;
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
@@ -546,8 +534,8 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -794,24 +794,17 @@ impl BucketTargetSys {
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
self.validate_target(bucket, target).await?;
|
||||
|
||||
let mut bucket_targets = self.targets_base_for_write(bucket).await?;
|
||||
let mut bucket_targets = match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
|
||||
|
||||
Ok(bucket_targets)
|
||||
}
|
||||
|
||||
/// Ordinary writes must not turn an unreadable cached snapshot into an
|
||||
/// empty configuration. Explicit repair belongs to the metadata transaction
|
||||
/// that can inspect the current persisted state.
|
||||
async fn targets_base_for_write(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Ok(targets),
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
|
||||
if !target.target_type.is_valid() {
|
||||
return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
|
||||
@@ -870,9 +863,7 @@ impl BucketTargetSys {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge a validated target into a caller-owned snapshot. The caller must
|
||||
/// protect that snapshot through persistence.
|
||||
pub fn upsert_target_entry(
|
||||
fn upsert_target_entry(
|
||||
bucket_targets: &mut Vec<BucketTarget>,
|
||||
target: &BucketTarget,
|
||||
update: bool,
|
||||
@@ -1236,27 +1227,25 @@ impl BucketTargetSys {
|
||||
return (String::new(), false);
|
||||
};
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default();
|
||||
Self::remote_arn_for_targets(targets, target, depl_id)
|
||||
}
|
||||
|
||||
/// Resolve create idempotency against the snapshot the caller will persist.
|
||||
pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
{
|
||||
let targets_map = self.targets_map.read().await;
|
||||
if let Some(targets) = targets_map.get(bucket) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4324,41 +4313,4 @@ mod tests {
|
||||
let window = LastMinuteLatency::new();
|
||||
assert_eq!(window.get_total().avg, Duration::from_secs(0));
|
||||
}
|
||||
|
||||
fn repair_target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
endpoint: "remote.example.com".to_string(),
|
||||
target_bucket: "remote".to_string(),
|
||||
arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unreadable_target_set_refuses_cached_writes() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-opt-in";
|
||||
sys.mark_targets_unreadable(bucket).await;
|
||||
assert!(matches!(
|
||||
sys.targets_base_for_write(bucket).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_readable_target_set_remains_the_write_base() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-readable";
|
||||
let existing = repair_target(bucket, "keep");
|
||||
sys.targets_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), vec![existing.clone()]);
|
||||
let base = sys.targets_base_for_write(bucket).await.expect("read targets");
|
||||
assert_eq!(base.targets.len(), 1);
|
||||
assert_eq!(base.targets[0].arn, existing.arn);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,18 +150,6 @@ static XXHASH_SEED: u64 = 0;
|
||||
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
struct FreeVersionPostRemoteDeleteTestBarrier {
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc<FreeVersionPostRemoteDeleteTestBarrier>;
|
||||
}
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -922,11 +910,6 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
})??;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
// Remote DELETE is idempotent, but a changed fence makes the local
|
||||
// outcome ambiguous. Keep every marker for a fully fenced retry.
|
||||
@@ -5848,7 +5831,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
@@ -7847,119 +7830,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_remove_waits_for_inflight_free_version_local_commit() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await;
|
||||
let tier_manager = ecstore.tier_config_mgr();
|
||||
{
|
||||
let manager = tier_manager.read().await;
|
||||
manager
|
||||
.save_tiering_config(Arc::clone(&ecstore))
|
||||
.await
|
||||
.expect("mock tier configuration should persist before removal");
|
||||
}
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await;
|
||||
let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("seeded free version should be listed");
|
||||
let oi = page
|
||||
.items
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("seeded free version should be recoverable");
|
||||
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
seed_lease
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("remote free-version tuple should be seeded");
|
||||
drop(seed_lease);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("free-version cleanup should pause after the remote delete");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
|
||||
let remove_manager = Arc::clone(&tier_manager);
|
||||
let remove_store = Arc::clone(&ecstore);
|
||||
let remove_backend = backend.clone();
|
||||
let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone())));
|
||||
let mut remove = tokio::spawn(async move {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(
|
||||
remove_driver_factory,
|
||||
TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let prepared = tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
loop {
|
||||
match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await {
|
||||
Ok(lease) => drop(lease),
|
||||
Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break,
|
||||
Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
tokio::select! {
|
||||
prepared = prepared => {
|
||||
prepared.expect("tier remove should install its prepared admission fence");
|
||||
}
|
||||
result = &mut remove => {
|
||||
panic!("tier remove finished before installing its prepared admission fence: {result:?}");
|
||||
}
|
||||
}
|
||||
assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit");
|
||||
|
||||
barrier.release.notify_one();
|
||||
tokio::time::timeout(StdDuration::from_secs(30), cleanup)
|
||||
.await
|
||||
.expect("free-version cleanup should finish after release")
|
||||
.expect("free-version cleanup task should join")
|
||||
.expect("free-version cleanup should keep its generation current");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), remove)
|
||||
.await
|
||||
.expect("tier remove should finish after local cleanup")
|
||||
.expect("tier remove task should join")
|
||||
.expect("tier remove should pass its fresh authoritative proof");
|
||||
|
||||
assert!(!tier_manager.read().await.is_tier_valid("WARM"));
|
||||
for disk_path in &disk_paths {
|
||||
assert!(
|
||||
!fs::try_exists(disk_path.join(&bucket).join(object))
|
||||
.await
|
||||
.expect("post-removal free-version path check should succeed")
|
||||
);
|
||||
}
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("empty free-version test bucket should be removed");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::object_api::ObjectInfo;
|
||||
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
|
||||
TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
|
||||
expiration_action_has_valid_target,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
|
||||
},
|
||||
manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
|
||||
manual_transition_job, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
@@ -41,7 +41,6 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
ManualTransitionScope,
|
||||
ManualTransitionTask,
|
||||
ManualTransitionWorkerResult,
|
||||
RecoveryControl,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -106,14 +105,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-control",
|
||||
prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
|
||||
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryControl,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
@@ -123,7 +116,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
RECOVERY_CONTROL_NAMESPACE,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -249,18 +241,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
ManualTransitionWorkerResult {
|
||||
content_sha256: String,
|
||||
},
|
||||
RecoveryControl {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
first_seen_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
classification: recovery_control::IlmRecoveryClassification,
|
||||
attempt_count: u64,
|
||||
consecutive_failure_count: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
owner_fence_sha256: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl DurableIlmRecordCheckpoint {
|
||||
@@ -274,8 +254,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 }
|
||||
| Self::RecoveryControl { content_sha256, .. } => content_sha256,
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,51 +528,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
..
|
||||
},
|
||||
) => previous_identity == next_identity && next_updated_at > previous_updated_at,
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
classification: previous_classification,
|
||||
attempt_count: previous_attempts,
|
||||
consecutive_failure_count: previous_failures,
|
||||
owner_fence_sha256: previous_owner,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: next_identity,
|
||||
source_generation_sha256: next_generation,
|
||||
first_seen_at_unix_nanos: next_first_seen,
|
||||
revision: next_revision,
|
||||
classification: next_classification,
|
||||
attempt_count: next_attempts,
|
||||
consecutive_failure_count: next_failures,
|
||||
owner_fence_sha256: next_owner,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
let adjacent = previous_identity == next_identity
|
||||
&& previous_first_seen == next_first_seen
|
||||
&& previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let claim = next_owner.is_some()
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts.checked_add(1) == Some(*next_attempts)
|
||||
&& previous_failures == next_failures;
|
||||
let source_refresh = previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts == next_attempts
|
||||
&& previous_failures == next_failures
|
||||
&& previous_generation != next_generation;
|
||||
let completion = previous_owner.is_some()
|
||||
&& next_owner.is_none()
|
||||
&& previous_generation == next_generation
|
||||
&& previous_attempts == next_attempts;
|
||||
adjacent && (claim || source_refresh || completion)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
@@ -619,14 +553,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryControl { classification, .. } = terminal
|
||||
&& !matches!(
|
||||
classification,
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -726,32 +652,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
|
||||
}
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
attempt_count: previous_attempts,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: terminal_identity,
|
||||
source_generation_sha256: terminal_generation,
|
||||
first_seen_at_unix_nanos: terminal_first_seen,
|
||||
revision: terminal_revision,
|
||||
attempt_count: terminal_attempts,
|
||||
classification:
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == terminal_identity
|
||||
&& (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
|
||||
&& previous_first_seen == terminal_first_seen
|
||||
&& terminal_revision > previous_revision
|
||||
&& terminal_attempts >= previous_attempts
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1319,35 +1219,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryControl => {
|
||||
let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control =
|
||||
recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
|
||||
let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
if canonical != path || control.identity.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery control path is not canonical"));
|
||||
}
|
||||
let identity_sha256 = checkpoint_hash(&control.identity)?;
|
||||
let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
|
||||
let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
|
||||
(
|
||||
"control_id",
|
||||
control_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryControl {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
source_generation_sha256,
|
||||
first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
|
||||
revision: control.revision,
|
||||
classification: control.classification,
|
||||
attempt_count: control.attempt_count,
|
||||
consecutive_failure_count: control.consecutive_failure_count,
|
||||
owner_fence_sha256,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1541,94 +1412,6 @@ mod tests {
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
|
||||
let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
|
||||
let generation = recovery_control::IlmRecoverySourceGeneration::new(
|
||||
transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
|
||||
"source-etag",
|
||||
"a".repeat(64),
|
||||
vec![recovery_control::IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.to_string(),
|
||||
etag: "source-etag".to_string(),
|
||||
encoded_len: 128,
|
||||
content_sha256: "a".repeat(64),
|
||||
}],
|
||||
)
|
||||
.expect("source generation should build");
|
||||
recovery_control::IlmRecoveryControl::new(
|
||||
recovery_control::IlmRecoveryControlIdentity {
|
||||
protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: source_path.to_string(),
|
||||
stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
},
|
||||
generation,
|
||||
recovery_control::IlmRecoveryClassification::Retrying,
|
||||
1_000_000_000,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("recovery control should build")
|
||||
}
|
||||
|
||||
fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
|
||||
let control_id = control.identity.source_operation_digest().expect("control id should derive");
|
||||
let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
|
||||
.expect("control path should build");
|
||||
let encoded = control.encode().expect("control should encode");
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery control namespace should classify")
|
||||
.expect("recovery control should be durable");
|
||||
assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("recovery control should validate")
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
|
||||
let initial_control = recovery_control_fixture();
|
||||
let initial = recovery_control_checkpoint(&initial_control);
|
||||
|
||||
let mut claimed_control = initial_control;
|
||||
let mut advanced_generation = claimed_control.observed_source_generation.clone();
|
||||
advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
|
||||
claimed_control
|
||||
.claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
|
||||
.expect("control should claim");
|
||||
let claimed = recovery_control_checkpoint(&claimed_control);
|
||||
initial.validate_successor(&claimed).expect("claim should advance receipt");
|
||||
|
||||
let mut retry_control = claimed_control;
|
||||
retry_control
|
||||
.record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("retry should persist");
|
||||
let retry = recovery_control_checkpoint(&retry_control);
|
||||
claimed.validate_successor(&retry).expect("retry should advance receipt");
|
||||
|
||||
let ready_at = retry_control
|
||||
.next_attempt_at_unix_nanos
|
||||
.expect("retry deadline should persist");
|
||||
let mut terminal_control = retry_control;
|
||||
terminal_control
|
||||
.claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
|
||||
.expect("retry should claim");
|
||||
let reclaimed = recovery_control_checkpoint(&terminal_control);
|
||||
retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
|
||||
terminal_control
|
||||
.finish_attempt(
|
||||
recovery_control::IlmRecoveryClassification::Terminal,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("control should terminate");
|
||||
let terminal = recovery_control_checkpoint(&terminal_control);
|
||||
reclaimed
|
||||
.validate_successor(&terminal)
|
||||
.expect("terminal state should advance receipt");
|
||||
assert!(initial.is_predecessor_of_terminal(&terminal));
|
||||
assert!(!initial.is_predecessor_of_terminal(&retry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
|
||||
let initial_intent = tier_probe_intent_fixture();
|
||||
|
||||
@@ -24,7 +24,6 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
|
||||
mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod recovery_control;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -35,10 +35,6 @@ use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::{
|
||||
TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record,
|
||||
};
|
||||
use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
|
||||
load_recovery_control, observe_recovery_source, save_recovery_control_if_absent,
|
||||
};
|
||||
use crate::bucket::lifecycle::runtime_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -82,13 +78,6 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
|
||||
const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
|
||||
const TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS: &str = "tier_delete_journal_corrupt";
|
||||
const CORRUPT_TIER_DELETE_JOURNAL_IDENTITY: &str = "corrupt";
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
|
||||
@@ -5520,125 +5509,6 @@ enum TierDeleteJournalEntryRecoveryOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&str> {
|
||||
let identity = object_name
|
||||
.strip_prefix(TIER_DELETE_JOURNAL_LEGACY_PREFIX)?
|
||||
.strip_suffix(".json")?;
|
||||
(rustfs_utils::crypto::is_sha256_checksum(identity)
|
||||
&& !identity
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()))
|
||||
.then_some(identity)
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
|
||||
match entry.persisted_version {
|
||||
1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
|
||||
TIER_DELETE_JOURNAL_VERSION => Some((TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_matches(
|
||||
control: &IlmRecoveryControl,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
generation: &crate::bucket::lifecycle::recovery_control::IlmRecoverySourceGeneration,
|
||||
classification: IlmRecoveryClassification,
|
||||
error_code: IlmRecoveryErrorCode,
|
||||
) -> bool {
|
||||
control.identity == *identity
|
||||
&& control.observed_source_generation == *generation
|
||||
&& control.classification == classification
|
||||
&& control.last_error_code == error_code
|
||||
&& control.owner.is_none()
|
||||
&& control.attempt_count == 0
|
||||
&& control.consecutive_failure_count == 0
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_is_scheduler_fence(control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity) -> bool {
|
||||
control.identity == *identity && control.owner.is_none() && !control.classification.permits_automatic_attempt()
|
||||
}
|
||||
|
||||
async fn persist_legacy_tier_delete_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
object_name: &str,
|
||||
observed_data: &[u8],
|
||||
stable_operation_identity: String,
|
||||
(source_schema, record_class): (&'static str, &'static str),
|
||||
intended_classification: IlmRecoveryClassification,
|
||||
intended_error_code: IlmRecoveryErrorCode,
|
||||
) -> Result<()> {
|
||||
let identity = IlmRecoveryControlIdentity {
|
||||
protocol: IlmRecoveryProtocol::TierDeleteJournal,
|
||||
canonical_source_path: object_name.to_string(),
|
||||
stable_operation_identity,
|
||||
record_class: record_class.to_string(),
|
||||
};
|
||||
let control_id = identity.source_operation_digest().map_err(Error::other)?;
|
||||
match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
|
||||
Ok(observed) if legacy_tier_delete_control_is_scheduler_fence(&observed.control, &identity) => return Ok(()),
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let source = observe_recovery_source(api.clone(), object_name, source_schema).await?;
|
||||
let exact_source = source.is_consistent() && source.canonical_data.as_deref() == Some(observed_data);
|
||||
let (classification, error_code) = if exact_source {
|
||||
(intended_classification, intended_error_code)
|
||||
} else {
|
||||
(IlmRecoveryClassification::Corrupt, IlmRecoveryErrorCode::SourceDivergent)
|
||||
};
|
||||
let candidate = IlmRecoveryControl::new(
|
||||
identity.clone(),
|
||||
source.generation.clone(),
|
||||
classification,
|
||||
i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.map_err(|_| Error::other("tier delete journal recovery timestamp does not fit i64"))?,
|
||||
error_code,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match save_recovery_control_if_absent(api.clone(), &candidate).await {
|
||||
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||
Err(save_error) => match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
|
||||
Ok(observed)
|
||||
if legacy_tier_delete_control_matches(
|
||||
&observed.control,
|
||||
&identity,
|
||||
&source.generation,
|
||||
classification,
|
||||
error_code,
|
||||
) =>
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Ok(_) | Err(_) => return Err(save_error),
|
||||
},
|
||||
}
|
||||
|
||||
let observed = load_recovery_control(api, IlmRecoveryProtocol::TierDeleteJournal, &control_id).await?;
|
||||
if !legacy_tier_delete_control_matches(&observed.control, &identity, &source.generation, classification, error_code) {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn retain_corrupt_legacy_tier_delete_journal(api: Arc<ECStore>, object_name: &str, data: &[u8]) -> Result<()> {
|
||||
canonical_legacy_tier_delete_journal_identity(object_name)
|
||||
.ok_or_else(|| Error::other("tier delete journal path is not canonical"))?;
|
||||
persist_legacy_tier_delete_recovery_control(
|
||||
api,
|
||||
object_name,
|
||||
data,
|
||||
CORRUPT_TIER_DELETE_JOURNAL_IDENTITY.to_string(),
|
||||
(TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS),
|
||||
IlmRecoveryClassification::Corrupt,
|
||||
IlmRecoveryErrorCode::SourceCorrupt,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome {
|
||||
let data = match config_boundary::read_config(api.clone(), &object_name).await {
|
||||
Ok(data) => data,
|
||||
@@ -5659,22 +5529,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
let je = match decode_tier_delete_journal_entry(&data) {
|
||||
Ok(je) => je,
|
||||
Err(err) => {
|
||||
if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
|
||||
return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(control_error) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?control_error,
|
||||
"Failed to retain corrupt tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5688,22 +5542,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
};
|
||||
|
||||
if tier_delete_journal_object_name(&je) != object_name {
|
||||
if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
|
||||
return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?err,
|
||||
"Failed to retain mismatched tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5714,36 +5552,6 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
|
||||
return TierDeleteJournalEntryRecoveryOutcome::Failed;
|
||||
}
|
||||
|
||||
if let Some((source_schema, record_class)) = legacy_tier_delete_recovery_descriptor(&je) {
|
||||
let stable_operation_identity = canonical_legacy_tier_delete_journal_identity(&object_name)
|
||||
.expect("decoded legacy journal path was validated against its canonical object name")
|
||||
.to_string();
|
||||
return match persist_legacy_tier_delete_recovery_control(
|
||||
api,
|
||||
&object_name,
|
||||
&data,
|
||||
stable_operation_identity,
|
||||
(source_schema, record_class),
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object_name,
|
||||
error = ?err,
|
||||
"Failed to retain legacy tier delete journal recovery control"
|
||||
);
|
||||
TierDeleteJournalEntryRecoveryOutcome::Failed
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
match api
|
||||
.durable_ilm_terminal_receipt_covers_active_source(&object_name, &data)
|
||||
.await
|
||||
|
||||
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -706,16 +706,15 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
|
||||
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
|
||||
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
|
||||
@@ -23,11 +23,6 @@ use uuid::Uuid;
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
|
||||
ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
|
||||
save_recovery_control_if_absent, save_recovery_control_if_current,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
@@ -49,7 +44,6 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
|
||||
pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
|
||||
const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
|
||||
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
|
||||
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
|
||||
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
|
||||
@@ -743,11 +737,9 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
RemoteCandidateDeleted,
|
||||
RecordDeleted,
|
||||
Retained,
|
||||
RetainedAmbiguous(IlmRecoveryErrorCode),
|
||||
OperatorRequired(IlmRecoveryErrorCode),
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryClaimBarrierState {
|
||||
transaction_id: Uuid,
|
||||
@@ -755,17 +747,17 @@ struct TransitionRecoveryClaimBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TransitionRecoveryClaimBarrier {
|
||||
state: Arc<TransitionRecoveryClaimBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl TransitionRecoveryClaimBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryClaimBarrierState {
|
||||
@@ -796,7 +788,7 @@ impl TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionRecoveryClaimBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -810,7 +802,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -825,80 +817,6 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryTerminalBarrierState {
|
||||
transaction_id: Uuid,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionRecoveryTerminalBarrier {
|
||||
state: Arc<TransitionRecoveryTerminalBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl TransitionRecoveryTerminalBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryTerminalBarrierState {
|
||||
transaction_id,
|
||||
..Default::default()
|
||||
});
|
||||
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison");
|
||||
assert!(
|
||||
slot.is_none(),
|
||||
"transition recovery terminal barrier must be installed by one test at a time"
|
||||
);
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("transition recovery should persist terminal control before source cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionRecoveryTerminalBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.transaction_id == transaction_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
@@ -1102,35 +1020,17 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
|
||||
let suffix = object
|
||||
.strip_prefix(&prefix)
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let shard_a = parts
|
||||
let file_name = suffix
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
let file_name = parts
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
|
||||
}
|
||||
let transaction_key = file_name
|
||||
.strip_suffix(".json")
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
|
||||
if transaction_key.len() != 32
|
||||
|| !transaction_key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
|| shard_a != &transaction_key[..2]
|
||||
|| shard_b != &transaction_key[2..4]
|
||||
{
|
||||
if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
|
||||
}
|
||||
Uuid::parse_str(transaction_key)
|
||||
.ok()
|
||||
.filter(|transaction_id| !transaction_id.is_nil())
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
|
||||
Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
|
||||
}
|
||||
|
||||
pub async fn process_transition_transaction_record(
|
||||
@@ -1155,27 +1055,6 @@ async fn process_transition_transaction_record_at(
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
let record_name =
|
||||
transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let now_unix_nanos =
|
||||
i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
|
||||
let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
|
||||
let recovery_control_id = recovery_control_identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control_record_name =
|
||||
recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
|
||||
Some(
|
||||
api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _control_guard = match &control_lock {
|
||||
Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
|
||||
None => None,
|
||||
};
|
||||
// The synthetic key avoids nesting the recovery lock with the config
|
||||
// object's own I/O lock. Holding it across the bounded source proof and
|
||||
// remote DELETE elects one destructive recovery worker across nodes.
|
||||
@@ -1194,400 +1073,55 @@ async fn process_transition_transaction_record_at(
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
|
||||
let mut recovery_control = if transition_state_needs_recovery_control(¤t, now_unix_nanos) {
|
||||
if cleanup_terminal_transition_recovery_control(
|
||||
api.clone(),
|
||||
¤t,
|
||||
&record_name,
|
||||
&recovery_control_identity,
|
||||
&recovery_control_id,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
|
||||
}
|
||||
match claim_transition_recovery_control(
|
||||
api.clone(),
|
||||
¤t,
|
||||
&record_name,
|
||||
recovery_control_identity,
|
||||
&recovery_control_id,
|
||||
now_unix_nanos,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Some(control) => Some(control),
|
||||
None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let recovery = match current.state {
|
||||
match current.state {
|
||||
TransitionTransactionState::Uploaded => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
let mut cleanup = current.clone();
|
||||
cleanup
|
||||
.mark_cleanup_pending(
|
||||
current.fence(),
|
||||
TransitionCleanupProof {
|
||||
transaction_id: current.transaction_id,
|
||||
write_id: current.write_id,
|
||||
remote_object: current.remote_object.clone(),
|
||||
remote_version: current.remote_version.clone(),
|
||||
backend_fingerprint: current.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_before_transition_recovery_claim(current.transaction_id).await;
|
||||
match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await {
|
||||
Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
|
||||
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
if transition_transaction_ownership_is_active(¤t, now_unix_nanos) {
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
let mut cleanup = current.clone();
|
||||
cleanup
|
||||
.mark_cleanup_pending(
|
||||
current.fence(),
|
||||
TransitionCleanupProof {
|
||||
transaction_id: current.transaction_id,
|
||||
write_id: current.write_id,
|
||||
remote_object: current.remote_object.clone(),
|
||||
remote_version: current.remote_version.clone(),
|
||||
backend_fingerprint: current.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
#[cfg(test)]
|
||||
pause_before_transition_recovery_claim(current.transaction_id).await;
|
||||
match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await {
|
||||
Ok(()) => recover_cleanup_pending(api, &cleanup).await,
|
||||
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), ¤t).await,
|
||||
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, ¤t).await,
|
||||
TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), ¤t).await {
|
||||
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
|
||||
IlmRecoveryErrorCode::LocalCommitAmbiguous,
|
||||
)),
|
||||
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
|
||||
IlmRecoveryErrorCode::LocalCommitAmbiguous,
|
||||
)),
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionTransactionState::UploadOutcomeUnknown => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
if transition_transaction_ownership_is_active(¤t, now_unix_nanos) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
recover_unknown_upload_outcome(api.clone(), ¤t).await
|
||||
recover_unknown_upload_outcome(api, ¤t).await
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::UploadStarted => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(mut control) = recovery_control.take() {
|
||||
let source_to_delete = if matches!(
|
||||
recovery,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
|
||||
| TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
) {
|
||||
let refreshed =
|
||||
refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
|
||||
control = refreshed.0;
|
||||
refreshed.1
|
||||
} else {
|
||||
None
|
||||
};
|
||||
persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
|
||||
if let Some(source) = source_to_delete {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_after_transition_recovery_terminal(source.transaction_id).await;
|
||||
delete_transition_transaction_record(api, &source).await?;
|
||||
}
|
||||
} else if matches!(
|
||||
recovery,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
) {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
}
|
||||
recovery
|
||||
}
|
||||
|
||||
fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
|
||||
IlmRecoveryControlIdentity {
|
||||
protocol: IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: record_name.to_string(),
|
||||
stable_operation_identity: transaction.transaction_id.to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
|
||||
let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
|
||||
transition_recovery_control_identity(transaction, &record_name)
|
||||
.source_operation_digest()
|
||||
.map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
|
||||
}
|
||||
|
||||
fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
|
||||
now_unix_nanos >= transaction.not_after_unix_nanos
|
||||
&& !matches!(
|
||||
transaction.state,
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
|
||||
)
|
||||
}
|
||||
|
||||
async fn cleanup_terminal_transition_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
record_name: &str,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
control_id: &str,
|
||||
) -> EcstoreResult<bool> {
|
||||
let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(observed) => observed,
|
||||
Err(Error::ConfigNotFound) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if observed.control.classification != IlmRecoveryClassification::Terminal {
|
||||
return Ok(false);
|
||||
}
|
||||
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source.generation == observed.control.observed_source_generation
|
||||
&& source.canonical_data.as_deref().is_some_and(|data| {
|
||||
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
|
||||
});
|
||||
if observed.control.identity != *identity || !exact_source {
|
||||
return Ok(false);
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn claim_transition_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
record_name: &str,
|
||||
identity: IlmRecoveryControlIdentity,
|
||||
control_id: &str,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
|
||||
let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(control) => Some(control),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if let Some(observed) = existing.as_ref() {
|
||||
if observed.control.identity != identity {
|
||||
return Ok(None);
|
||||
}
|
||||
if observed
|
||||
.control
|
||||
.owner
|
||||
.as_ref()
|
||||
.is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos)
|
||||
{
|
||||
let mut expired = observed.control.clone();
|
||||
expired
|
||||
.record_expired_attempt(now_unix_nanos)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, observed, &expired).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
if !observed.control.should_attempt_at(now_unix_nanos) {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
|
||||
Ok(source) => source,
|
||||
Err(err) => {
|
||||
if let Some(observed) = existing {
|
||||
persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let source_matches = source.is_consistent()
|
||||
&& source.canonical_data.as_deref().is_some_and(|data| {
|
||||
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
|
||||
});
|
||||
let source_error = if source_matches {
|
||||
IlmRecoveryErrorCode::None
|
||||
} else if source.canonical_data.is_some() {
|
||||
IlmRecoveryErrorCode::SourceGenerationChanged
|
||||
} else {
|
||||
IlmRecoveryErrorCode::SourceDivergent
|
||||
};
|
||||
|
||||
let mut observed = match existing {
|
||||
Some(control) => control,
|
||||
None => {
|
||||
let candidate = IlmRecoveryControl::new(
|
||||
identity.clone(),
|
||||
source.generation.clone(),
|
||||
if source_matches {
|
||||
IlmRecoveryClassification::Retrying
|
||||
} else {
|
||||
IlmRecoveryClassification::Corrupt
|
||||
},
|
||||
now_unix_nanos,
|
||||
source_error,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
match save_recovery_control_if_absent(api.clone(), &candidate).await {
|
||||
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
|
||||
}
|
||||
};
|
||||
if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut claimed = observed.control.clone();
|
||||
claimed
|
||||
.claim_for_source_generation(
|
||||
api.id.to_string(),
|
||||
Uuid::new_v4(),
|
||||
now_unix_nanos,
|
||||
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
|
||||
source.generation,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
|
||||
observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
|
||||
if observed.control != claimed {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if !source_matches {
|
||||
let mut corrupt = observed.control.clone();
|
||||
corrupt
|
||||
.finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, &observed, &corrupt).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(observed))
|
||||
}
|
||||
|
||||
async fn persist_transition_recovery_source_failure(
|
||||
api: Arc<ECStore>,
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<()> {
|
||||
let mut claimed = observed.control.clone();
|
||||
claimed
|
||||
.claim(
|
||||
api.id.to_string(),
|
||||
Uuid::new_v4(),
|
||||
now_unix_nanos,
|
||||
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
|
||||
let claimed = load_recovery_control(
|
||||
api.clone(),
|
||||
IlmRecoveryProtocol::TransitionTransaction,
|
||||
&claimed
|
||||
.identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
)
|
||||
.await?;
|
||||
let mut failed = claimed.control.clone();
|
||||
failed
|
||||
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, &claimed, &failed).await
|
||||
}
|
||||
|
||||
async fn refresh_transition_recovery_control_source(
|
||||
api: Arc<ECStore>,
|
||||
mut observed: ObservedIlmRecoveryControl,
|
||||
record_name: &str,
|
||||
transaction_id: Uuid,
|
||||
) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
|
||||
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
|
||||
Ok(transaction) => transaction,
|
||||
Err(Error::ConfigNotFound) => return Ok((observed, None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source
|
||||
.canonical_data
|
||||
.as_deref()
|
||||
.is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
|
||||
if !exact_source {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if observed.control.observed_source_generation != source.generation {
|
||||
let mut refreshed = observed.control.clone();
|
||||
refreshed
|
||||
.refresh_owned_source_generation(source.generation)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
|
||||
observed = load_recovery_control(
|
||||
api,
|
||||
IlmRecoveryProtocol::TransitionTransaction,
|
||||
&refreshed
|
||||
.identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
)
|
||||
.await?;
|
||||
if observed.control != refreshed {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
Ok((observed, Some(transaction)))
|
||||
}
|
||||
|
||||
async fn persist_transition_recovery_result(
|
||||
api: Arc<ECStore>,
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<()> {
|
||||
let mut next = observed.control.clone();
|
||||
match recovery {
|
||||
Ok(
|
||||
TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
|
||||
) => next
|
||||
.finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained) => next
|
||||
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
|
||||
.finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
|
||||
.finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Err(err) => next
|
||||
.record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
}
|
||||
save_recovery_control_if_current(api, &observed, &next).await
|
||||
}
|
||||
|
||||
fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
|
||||
match err {
|
||||
Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
|
||||
Error::ConfigNotFound
|
||||
| Error::FileNotFound
|
||||
| Error::FileVersionNotFound
|
||||
| Error::ObjectNotFound(_, _)
|
||||
| Error::VersionNotFound(_, _, _)
|
||||
| Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
|
||||
Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
|
||||
_ => IlmRecoveryErrorCode::Unknown,
|
||||
TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1600,7 +1134,10 @@ async fn recover_cleanup_pending(
|
||||
transaction: &TransitionTransaction,
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
match local_commit_matches_transaction(api.clone(), transaction).await {
|
||||
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
|
||||
Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
|
||||
Err(err) => Err(err),
|
||||
@@ -1620,6 +1157,7 @@ async fn delete_unreferenced_transition_candidate(
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
delete_transition_remote_candidate(api.clone(), ¤t).await?;
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
|
||||
@@ -1640,26 +1178,24 @@ async fn recover_unknown_upload_outcome(
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
TransitionCandidateProbe::Missing => {
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
))
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteProbeAmbiguous,
|
||||
)),
|
||||
TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteProbeUnsupported,
|
||||
)),
|
||||
TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1753,7 +1289,7 @@ pub async fn recover_transition_transaction_records(
|
||||
recover_transition_transaction_records_with_now(api, limit, marker, None).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn recover_transition_transaction_records_at(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -1787,11 +1323,6 @@ async fn recover_transition_transaction_records_with_now(
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
if list.is_truncated && list.next_continuation_token.is_none() {
|
||||
return Err(Error::other(
|
||||
"transition transaction recovery returned a truncated page without a continuation marker",
|
||||
));
|
||||
}
|
||||
|
||||
let mut stats = TransitionTransactionRecoveryStats {
|
||||
scanned: 0,
|
||||
@@ -1850,11 +1381,7 @@ async fn recover_transition_transaction_records_with_now(
|
||||
) => {
|
||||
stats.recovered += 1;
|
||||
}
|
||||
Ok(
|
||||
TransitionTransactionRecoveryOutcome::Retained
|
||||
| TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
|
||||
| TransitionTransactionRecoveryOutcome::OperatorRequired(_),
|
||||
) => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained) => {
|
||||
stats.retained += 1;
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
|
||||
@@ -1982,74 +1509,11 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
const BACKEND_FINGERPRINT: [u8; 32] = [7; 32];
|
||||
|
||||
struct RecoveryAttemptDropGuard(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for RecoveryAttemptDropGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
async fn pending_recovery_attempt(started: Arc<tokio::sync::Notify>, dropped: Arc<AtomicBool>) -> EcstoreResult<()> {
|
||||
let _drop_guard = RecoveryAttemptDropGuard(dropped);
|
||||
started.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() {
|
||||
let timeout_started = Arc::new(tokio::sync::Notify::new());
|
||||
let timeout_dropped = Arc::new(AtomicBool::new(false));
|
||||
let timeout_task = tokio::spawn({
|
||||
let started = Arc::clone(&timeout_started);
|
||||
let dropped = Arc::clone(&timeout_dropped);
|
||||
async move {
|
||||
await_transition_transaction_recovery(
|
||||
&CancellationToken::new(),
|
||||
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
|
||||
pending_recovery_attempt(started, dropped),
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
timeout_started.notified().await;
|
||||
tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await;
|
||||
let timed_out = timeout_task.await.expect("timeout wrapper task should join");
|
||||
assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass");
|
||||
assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt");
|
||||
|
||||
let cancel_token = CancellationToken::new();
|
||||
let cancel_started = Arc::new(tokio::sync::Notify::new());
|
||||
let cancel_dropped = Arc::new(AtomicBool::new(false));
|
||||
let cancel_task = tokio::spawn({
|
||||
let cancel_token = cancel_token.clone();
|
||||
let started = Arc::clone(&cancel_started);
|
||||
let dropped = Arc::clone(&cancel_dropped);
|
||||
async move {
|
||||
await_transition_transaction_recovery(
|
||||
&cancel_token,
|
||||
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
|
||||
pending_recovery_attempt(started, dropped),
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
cancel_started.notified().await;
|
||||
cancel_token.cancel();
|
||||
let cancelled = cancel_task.await.expect("cancellation wrapper task should join");
|
||||
assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop");
|
||||
assert!(
|
||||
cancel_dropped.load(Ordering::SeqCst),
|
||||
"outer cancellation must drop its in-flight attempt"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MemoryTransactionStore {
|
||||
records: HashMap<Uuid, Vec<u8>>,
|
||||
@@ -2504,19 +1968,5 @@ mod tests {
|
||||
transition_transaction_record_object_name(Uuid::nil()),
|
||||
Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
|
||||
));
|
||||
assert_eq!(
|
||||
transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
|
||||
transaction_id
|
||||
);
|
||||
for malformed in [
|
||||
object.to_ascii_uppercase(),
|
||||
object.replace("/aa/aa/", "/ff/aa/"),
|
||||
object.replace("/aa/aa/", "/aa/aa/extra/"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
transition_transaction_id_from_record_object_name(&malformed),
|
||||
Err(TransitionTransactionError::Corrupt(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -791,22 +791,9 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace one config payload and stamp its `*_config_updated_at` with the
|
||||
/// local clock. This is the entry for edits that originate here: the
|
||||
/// local write time is the edit's source time.
|
||||
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
self.update_config_at(config_file, data, OffsetDateTime::now_utc())
|
||||
}
|
||||
let updated = OffsetDateTime::now_utc();
|
||||
|
||||
/// [`Self::update_config`] with an explicit `updated_at` stamp.
|
||||
///
|
||||
/// For a config replicated from another site the edit's source time is
|
||||
/// the peer's `updated_at`, not the moment it lands here: staleness of
|
||||
/// the next incoming item is judged against the stored stamp, so stamping
|
||||
/// the local apply time would reject a newer source edit that was merely
|
||||
/// delivered late (backlog#2292). Only replication receivers should pass
|
||||
/// a foreign time; local edits keep [`Self::update_config`].
|
||||
pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
match config_file {
|
||||
BUCKET_POLICY_CONFIG => {
|
||||
self.policy_config_json = data;
|
||||
@@ -1538,39 +1525,6 @@ mod test {
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation);
|
||||
}
|
||||
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
let mut metadata = BucketMetadata::new("source-stamped");
|
||||
|
||||
let stamped = metadata
|
||||
.update_config_at(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), source_time)
|
||||
.unwrap();
|
||||
assert_eq!(stamped, source_time);
|
||||
assert_eq!(metadata.policy_config_updated_at, source_time);
|
||||
|
||||
let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
|
||||
let stamped = metadata
|
||||
.update_config_at(BUCKET_TAGGING_CONFIG, tagging, source_time)
|
||||
.unwrap();
|
||||
assert_eq!(stamped, source_time);
|
||||
assert_eq!(metadata.tagging_config_updated_at, source_time);
|
||||
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = metadata
|
||||
.update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec())
|
||||
.unwrap();
|
||||
assert!(stamped >= before, "a local edit is stamped with the local clock");
|
||||
assert_eq!(metadata.policy_config_updated_at, stamped);
|
||||
assert_eq!(
|
||||
metadata.tagging_config_updated_at, source_time,
|
||||
"restamping one config must not move another config's stamp"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_locking_requires_lock_metadata_not_plain_versioning() {
|
||||
use s3s::dto::ObjectLockEnabled;
|
||||
@@ -1657,34 +1611,6 @@ mod test {
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
|
||||
/// already carries as a compatibility fixture stores
|
||||
/// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
|
||||
/// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
|
||||
/// bytes below are the exact payload the fixture in
|
||||
/// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
|
||||
/// ever grows the array-shaped compatibility parse, this test is where the
|
||||
/// upgrade break is pinned and where the decision has to be recorded.
|
||||
#[test]
|
||||
fn minio_array_shaped_bucket_targets_are_unreadable() {
|
||||
let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
|
||||
let mut bm = BucketMetadata::new("minio-array-targets");
|
||||
bm.bucket_targets_config_json = minio_array.clone();
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a MinIO-shaped targets blob must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
bm.bucket_targets_unreadable(),
|
||||
"an array-shaped MinIO targets blob is unreadable, not an empty target set"
|
||||
);
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
assert_eq!(
|
||||
bm.bucket_targets_config_json, minio_array,
|
||||
"the raw MinIO bytes must survive so the configuration stays recoverable"
|
||||
);
|
||||
}
|
||||
|
||||
/// The invariant every branch of `parse_all_configs` shares: a stored but
|
||||
/// undecodable payload keeps its raw bytes and leaves the typed field
|
||||
/// `None`, so no branch fabricates a value. What a reader may then do with
|
||||
|
||||
@@ -567,32 +567,6 @@ pub async fn update_if_incarnation(
|
||||
config_file,
|
||||
data,
|
||||
Some(expected_incarnation_id),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`update_if_incarnation`] stamping the config with `updated_at` instead of
|
||||
/// the local clock.
|
||||
///
|
||||
/// For a site-replication receiver the edit's source time is the peer's
|
||||
/// `updated_at`; persisting it keeps the stored `*_config_updated_at` on the
|
||||
/// source clock so the next item's staleness is judged source-time against
|
||||
/// source-time (backlog#2292). See [`BucketMetadata::update_config_at`].
|
||||
pub async fn update_if_incarnation_at(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(update_with_sys_expected(
|
||||
get_bucket_metadata_sys()?,
|
||||
bucket,
|
||||
config_file,
|
||||
data,
|
||||
Some(expected_incarnation_id),
|
||||
Some(updated_at),
|
||||
))
|
||||
.await
|
||||
}
|
||||
@@ -603,30 +577,6 @@ pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_inc
|
||||
bucket,
|
||||
config_file,
|
||||
Some(expected_incarnation_id),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`delete_if_incarnation`] stamping the cleared config with `updated_at`
|
||||
/// (a replicated deletion's source time) instead of the local clock.
|
||||
///
|
||||
/// The stamp survives the deletion as the config's `*_config_updated_at`, and
|
||||
/// that is what the next incoming item is judged against: a local stamp on
|
||||
/// the delete would reject a newer source re-create that was merely delivered
|
||||
/// later (backlog#2292). See [`update_if_incarnation_at`].
|
||||
pub async fn delete_if_incarnation_at(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(delete_with_sys_expected(
|
||||
get_bucket_metadata_sys()?,
|
||||
bucket,
|
||||
config_file,
|
||||
Some(expected_incarnation_id),
|
||||
Some(updated_at),
|
||||
))
|
||||
.await
|
||||
}
|
||||
@@ -648,41 +598,34 @@ async fn update_with_sys(
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_with_sys_expected(sys, bucket, config_file, data, None, None).await
|
||||
update_with_sys_expected(sys, bucket, config_file, data, None).await
|
||||
}
|
||||
|
||||
/// `updated_at` is the stamp persisted on the config; `None` uses the local
|
||||
/// clock (the edit originates here), `Some` carries a replicated edit's
|
||||
/// source time (backlog#2292).
|
||||
async fn update_with_sys_expected(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, updated_at).await
|
||||
update_under_config_write_guard(sys, &guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// [`delete`] against an explicitly supplied metadata system. See
|
||||
/// [`update_with_sys`].
|
||||
async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys_expected(sys, bucket, config_file, None, None).await
|
||||
delete_with_sys_expected(sys, bucket, config_file, None).await
|
||||
}
|
||||
|
||||
/// `updated_at`: `None` stamps the local clock; `Some` persists a replicated
|
||||
/// deletion's source time (backlog#2292).
|
||||
async fn delete_with_sys_expected(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
delete_under_config_write_guard(sys, &guard, config_file, updated_at).await
|
||||
delete_under_config_write_guard(sys, &guard, config_file).await
|
||||
}
|
||||
|
||||
/// Owns the complete bucket-config mutation fence.
|
||||
@@ -829,21 +772,7 @@ pub async fn update_under_transaction_lock(
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, None).await
|
||||
}
|
||||
|
||||
/// [`update_under_transaction_lock`] stamping the config with `updated_at`
|
||||
/// (a replicated edit's source time) instead of the local clock; see
|
||||
/// [`update_if_incarnation_at`] (backlog#2292).
|
||||
pub async fn update_under_transaction_lock_at(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, Some(updated_at)).await
|
||||
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
|
||||
}
|
||||
|
||||
/// Clear one config file while the caller holds this bucket's transaction lock.
|
||||
@@ -853,7 +782,7 @@ pub async fn delete_under_transaction_lock(
|
||||
config_file: &str,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(bucket)?;
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, None).await
|
||||
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
|
||||
}
|
||||
|
||||
pub async fn update_quota_if_incarnation(
|
||||
@@ -861,29 +790,6 @@ pub async fn update_quota_if_incarnation(
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, None).await
|
||||
}
|
||||
|
||||
/// [`update_quota_if_incarnation`] stamping the quota config with
|
||||
/// `updated_at` (a replicated edit's source time) instead of the local
|
||||
/// clock; see [`update_if_incarnation_at`] (backlog#2292).
|
||||
pub async fn update_quota_if_incarnation_at(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, Some(updated_at)).await
|
||||
}
|
||||
|
||||
async fn update_quota_if_incarnation_stamped(
|
||||
bucket: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let guard = Box::pin(acquire_config_write_guard_for_incarnation(
|
||||
@@ -901,7 +807,7 @@ async fn update_quota_if_incarnation_stamped(
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data, updated_at).await
|
||||
update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(
|
||||
@@ -917,7 +823,6 @@ async fn update_under_config_write_guard(
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(&guard.bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
@@ -929,7 +834,7 @@ async fn update_under_config_write_guard(
|
||||
Some(&guard.transaction_guard),
|
||||
&guard.bucket,
|
||||
"bucket config transaction",
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id, updated_at),
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
@@ -941,7 +846,6 @@ async fn delete_under_config_write_guard(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
guard: &BucketMetadataMutationGuard,
|
||||
config_file: &str,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
guard.ensure_valid(&guard.bucket)?;
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
@@ -953,7 +857,7 @@ async fn delete_under_config_write_guard(
|
||||
Some(&guard.transaction_guard),
|
||||
&guard.bucket,
|
||||
"bucket config deletion transaction",
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id, updated_at),
|
||||
metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
@@ -1858,17 +1762,15 @@ impl BucketMetadataSys {
|
||||
/// `update` and the config read alone). Keep these boxed.
|
||||
pub async fn update(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?;
|
||||
Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id, None)).await
|
||||
Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await
|
||||
}
|
||||
|
||||
pub async fn delete(&self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
let incarnation_id = self.get_bucket_incarnation_id(bucket).await?;
|
||||
self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id, None)
|
||||
self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `updated_at`: `None` stamps the local clock; `Some` persists a
|
||||
/// replicated edit's source time (backlog#2292).
|
||||
async fn update_checked(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1876,7 +1778,6 @@ impl BucketMetadataSys {
|
||||
data: Vec<u8>,
|
||||
parse: bool,
|
||||
expected_incarnation_id: Uuid,
|
||||
updated_at: Option<OffsetDateTime>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
// Load through this system's own store, the one `save` persists to
|
||||
// (backlog#1052 S7). Reading from the ambient handle instead made the
|
||||
@@ -1887,10 +1788,7 @@ impl BucketMetadataSys {
|
||||
return Err(Error::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
|
||||
let updated = match updated_at {
|
||||
Some(updated_at) => bm.update_config_at(config_file, data, updated_at)?,
|
||||
None => bm.update_config(config_file, data)?,
|
||||
};
|
||||
let updated = bm.update_config(config_file, data)?;
|
||||
|
||||
Box::pin(self.save(bm)).await?;
|
||||
|
||||
@@ -3857,106 +3755,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2292: the explicit-stamp write path persists the given source
|
||||
/// time as the config's `*_config_updated_at` — through the incarnation
|
||||
/// path and through an already-held transaction guard — and survives a
|
||||
/// reload from disk, while the plain path keeps stamping the local clock.
|
||||
#[tokio::test]
|
||||
async fn explicit_updated_at_is_persisted_as_the_config_stamp() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "source-stamped-config";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
let source_time = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
|
||||
let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
|
||||
|
||||
// Incarnation path (`update_if_incarnation_at` minus the ambient lookup).
|
||||
let stamped =
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(source_time))
|
||||
.await
|
||||
.expect("source-stamped policy write should persist");
|
||||
assert_eq!(stamped, source_time);
|
||||
|
||||
// Held-guard path (`update_under_transaction_lock_at` minus the ambient lookup).
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await.expect("write guard");
|
||||
let stamped = update_under_config_write_guard(sys.clone(), &guard, BUCKET_TAGGING_CONFIG, tagging, Some(source_time))
|
||||
.await
|
||||
.expect("source-stamped tagging write should persist");
|
||||
drop(guard);
|
||||
assert_eq!(stamped, source_time);
|
||||
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
metadata_sys.metadata_map.write().await.clear();
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, source_time);
|
||||
assert_eq!(reloaded.tagging_config_updated_at, source_time);
|
||||
|
||||
// The plain path is unchanged: a local edit is stamped with the local clock.
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = update_with_sys(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy)
|
||||
.await
|
||||
.expect("locally stamped policy write should persist");
|
||||
assert!(stamped >= before, "the plain write path must keep stamping the local clock");
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, stamped);
|
||||
assert_eq!(
|
||||
reloaded.tagging_config_updated_at, source_time,
|
||||
"an unrelated config keeps its source stamp"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2292: a replicated delete persists the source time as the
|
||||
/// cleared config's `*_config_updated_at`, so the receive-side gate
|
||||
/// (source time against stored stamp) lets a newer source re-create land
|
||||
/// even when the delete was applied later than the re-create's source
|
||||
/// time; the plain delete keeps stamping the local clock.
|
||||
#[tokio::test]
|
||||
async fn explicit_updated_at_is_persisted_by_a_delete() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "source-stamped-delete";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
|
||||
let created_at = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
|
||||
let deleted_at = created_at + Duration::from_secs(60);
|
||||
let recreated_at = deleted_at + Duration::from_secs(60);
|
||||
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(created_at))
|
||||
.await
|
||||
.expect("source-stamped policy write should persist");
|
||||
let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, Some(deleted_at))
|
||||
.await
|
||||
.expect("source-stamped policy delete should persist");
|
||||
assert_eq!(stamped, deleted_at);
|
||||
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
metadata_sys.metadata_map.write().await.clear();
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert!(reloaded.policy_config_json.is_empty(), "the delete cleared the payload");
|
||||
assert_eq!(reloaded.policy_config_updated_at, deleted_at, "the delete kept the source stamp");
|
||||
assert!(
|
||||
recreated_at >= reloaded.policy_config_updated_at,
|
||||
"a re-create newer than the delete's source time is not stale against the stored stamp"
|
||||
);
|
||||
|
||||
// The plain delete path is unchanged: stamped with the local clock.
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy, None, Some(recreated_at))
|
||||
.await
|
||||
.expect("re-create should persist");
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, None)
|
||||
.await
|
||||
.expect("locally stamped delete should persist");
|
||||
assert!(stamped >= before, "the plain delete path must keep stamping the local clock");
|
||||
let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
|
||||
assert_eq!(reloaded.policy_config_updated_at, stamped);
|
||||
}
|
||||
|
||||
/// The load and the persisted write share one write guard, so concurrent
|
||||
/// rewrites of the same config compose instead of clobbering each other.
|
||||
/// Moving the load outside that guard loses all but the last tag.
|
||||
@@ -4173,16 +3971,10 @@ mod tests {
|
||||
let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
|
||||
assert_ne!(old_incarnation, new_incarnation);
|
||||
|
||||
let err = update_with_sys_expected(
|
||||
sys.clone(),
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
b"<Tagging/>".to_vec(),
|
||||
Some(old_incarnation),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("a request authorized for the deleted incarnation must fail closed");
|
||||
let err =
|
||||
update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec(), Some(old_incarnation))
|
||||
.await
|
||||
.expect_err("a request authorized for the deleted incarnation must fail closed");
|
||||
assert!(matches!(err, Error::BucketNotFound(name) if name == bucket));
|
||||
|
||||
let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap();
|
||||
@@ -4217,7 +4009,7 @@ mod tests {
|
||||
}],
|
||||
})
|
||||
.unwrap();
|
||||
update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging, None)
|
||||
update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!delete.is_finished());
|
||||
|
||||
@@ -20,9 +20,9 @@ pub use rustfs_replication::{
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_target_version_id, heal_uses_delete_replication_path,
|
||||
is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
|
||||
resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
|
||||
resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
|
||||
single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -882,20 +882,6 @@ fn reconstructed_heal_delete_info(
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
// The caller hands us a blank ObjectInfo (the source marker may already be
|
||||
// gone), so the state above carries no target-assigned marker version ids.
|
||||
// Restore them from the journal: `delete_marker_purge_version_id` must hit
|
||||
// the id the target reported, not fall back to the source marker id, which
|
||||
// a target that mints its own ids answers with an idempotent 204 that would
|
||||
// acknowledge the intent while the real marker stays behind (backlog#2290).
|
||||
// The corrupt flag rides along so a refusal stays a refusal after restart.
|
||||
for (arn, version_id) in &entry.target_delete_marker_version_ids {
|
||||
rstate
|
||||
.target_delete_marker_version_ids
|
||||
.entry(arn.clone())
|
||||
.or_insert_with(|| version_id.clone());
|
||||
}
|
||||
rstate.target_delete_marker_version_ids_corrupt |= entry.target_delete_marker_version_ids_corrupt;
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
@@ -6615,87 +6601,4 @@ mod tests {
|
||||
replacement_data
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#2290: a delete-marker purge intent that survives a restart
|
||||
/// through the MRF journal addresses the marker version the TARGET
|
||||
/// assigned, exactly as the live watcher does (see the
|
||||
/// `requires_delayed_purge` spawn). The journal carries the per-ARN ids
|
||||
/// (`targetDeleteMarkerVersionIDs`) and replay restores them into the
|
||||
/// reconstructed replication state; without that the replay would fall
|
||||
/// back to the source marker id, which a target that mints its own ids
|
||||
/// answers with an idempotent 204 — the entry would be acknowledged while
|
||||
/// the real marker stayed behind.
|
||||
#[test]
|
||||
fn mrf_delete_marker_purge_replay_preserves_target_assigned_marker_version() {
|
||||
use super::super::replication_object_decision_boundary::{delete_marker_purge_mrf_entry, delete_marker_purge_version_id};
|
||||
|
||||
let arn = "arn:minio:replication::generic-target:photos".to_string();
|
||||
let source_marker = uuid::Uuid::new_v4();
|
||||
let remote_marker = "remote-assigned-marker-version".to_string();
|
||||
|
||||
let live_oi = ObjectInfo {
|
||||
bucket: "photos".to_string(),
|
||||
name: "obj".to_string(),
|
||||
version_id: Some(source_marker),
|
||||
delete_marker: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut live_state = live_oi.replication_state();
|
||||
live_state.replicate_decision_str = replicate_decision_for_admitted_targets(std::slice::from_ref(&arn)).to_string();
|
||||
live_state
|
||||
.target_delete_marker_version_ids
|
||||
.insert(arn.clone(), remote_marker.clone());
|
||||
let live = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "obj".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(source_marker),
|
||||
replication_state: Some(live_state),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "photos".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(live.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
Some(Some(remote_marker.clone())),
|
||||
"the live purge addresses the recorded target version"
|
||||
);
|
||||
|
||||
// Watch window exhausted: persist the intent, restart, replay it.
|
||||
let entry = delete_marker_purge_mrf_entry(&live, vec![arn.clone()]);
|
||||
let replay_oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
|
||||
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
Some(Some(remote_marker)),
|
||||
"the MRF replay must address the target-assigned marker version, not source marker {source_marker}"
|
||||
);
|
||||
|
||||
// A refusal (inconsistent recorded ids) must stay a refusal across the
|
||||
// journal round trip instead of degrading into the source-id fallback.
|
||||
let mut refused = live;
|
||||
refused
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("state was set above")
|
||||
.target_delete_marker_version_ids_corrupt = true;
|
||||
let entry = delete_marker_purge_mrf_entry(&refused, vec![arn.clone()]);
|
||||
assert!(entry.target_delete_marker_version_ids_corrupt);
|
||||
let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
|
||||
None,
|
||||
"the MRF replay must keep refusing to guess when the recorded ids were inconsistent"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,11 +32,11 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec;
|
||||
use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, delete_replication_target_version_id, heal_uses_delete_replication_path,
|
||||
is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
|
||||
resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
target_delete_version_id,
|
||||
};
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -2051,11 +2051,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
// The watcher exists to purge a replicated marker once the SOURCE marker
|
||||
// vanishes. A version purge is that purge already (its failures reach the
|
||||
// journal as a purge entry), so it must not spawn a second watcher that
|
||||
// journals a duplicate intent (backlog#2290).
|
||||
let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object) && !is_version_purge;
|
||||
let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
|
||||
|
||||
let (replication_status, prev_status) = if !is_version_purge {
|
||||
(
|
||||
@@ -2765,6 +2761,12 @@ fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
} else {
|
||||
dobj.delete_object.version_id.unwrap_or_default()
|
||||
};
|
||||
|
||||
let mut rinfo = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
@@ -2797,25 +2799,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
// Purging a replicated delete marker addresses the version the target
|
||||
// assigned (recorded when the marker was created there); see
|
||||
// `delete_replication_target_version_id`. A corrupt record is a failure,
|
||||
// not a guess: the entry stays visible until the metadata is repaired.
|
||||
let Some(version_id) = delete_replication_target_version_id(&dobj.delete_object, &tgt_client.arn) else {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = %tgt_client.arn,
|
||||
reason = "recorded_target_version_inconsistent",
|
||||
"Replicated version purge refused: recorded target delete-marker version metadata is inconsistent"
|
||||
);
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Failed;
|
||||
rinfo.error = Some("recorded target delete-marker version metadata is inconsistent".to_string());
|
||||
return rinfo;
|
||||
};
|
||||
let version_id = target_delete_version_id(version_id, is_version_purge);
|
||||
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_for_worker(
|
||||
|
||||
@@ -68,10 +68,7 @@ use std::{
|
||||
},
|
||||
time::SystemTime,
|
||||
};
|
||||
use tokio::{
|
||||
net::TcpStream,
|
||||
time::{Duration, timeout},
|
||||
};
|
||||
use tokio::{net::TcpStream, time::Duration};
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -877,16 +874,6 @@ impl PeerRestClient {
|
||||
self.offline.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Prepare a retry without allowing connection-cache cleanup to extend the
|
||||
/// caller's probe deadline. The offline gate is cleared even when eviction
|
||||
/// times out so a cancelled cleanup cannot strand the peer in fast-fail
|
||||
/// mode; a later request can perform a fresh eviction if needed.
|
||||
pub async fn prepare_retry_with_timeout(&self, timeout_duration: Duration) -> bool {
|
||||
let evicted = timeout(timeout_duration, self.evict_connection()).await.is_ok();
|
||||
self.offline.store(false, Ordering::Release);
|
||||
evicted
|
||||
}
|
||||
|
||||
/// Whether this failure means the peer is unreachable, so it should be
|
||||
/// gated offline and its connection evicted.
|
||||
///
|
||||
|
||||
@@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
|
||||
let id_kind = parts
|
||||
.next()
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id"))
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
|
||||
let source_path = parts
|
||||
.next()
|
||||
@@ -3023,9 +3023,8 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
|
||||
}
|
||||
match id_kind {
|
||||
"operation_id" | "control_id" if !is_sha256_checksum(id) => {
|
||||
let id_label = id_kind.trim_end_matches("_id");
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
|
||||
"operation_id" if !is_sha256_checksum(id) => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id")));
|
||||
}
|
||||
"transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
|
||||
@@ -5108,49 +5107,7 @@ async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<Po
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
|
||||
let batch = uuid::Uuid::new_v4();
|
||||
for (pool, read) in reads.iter().enumerate() {
|
||||
let mut observation = serde_json::json!({
|
||||
"kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
|
||||
"cas": match &read.cas {
|
||||
PoolMetaCasToken::Missing => "missing",
|
||||
PoolMetaCasToken::Existing(_) => "existing",
|
||||
PoolMetaCasToken::Unsafe => "unsafe",
|
||||
},
|
||||
"etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
|
||||
});
|
||||
match &read.replica {
|
||||
PoolMetaReplica::Valid {
|
||||
raw,
|
||||
canonical,
|
||||
meta,
|
||||
revision,
|
||||
committed,
|
||||
..
|
||||
} => {
|
||||
observation["state"] = serde_json::json!("valid");
|
||||
observation["committed"] = serde_json::json!(committed);
|
||||
observation["version"] = serde_json::json!(revision.version);
|
||||
observation["cluster_id"] = serde_json::json!(revision.cluster_id);
|
||||
observation["epoch"] = serde_json::json!(revision.epoch);
|
||||
observation["generation"] = serde_json::json!(revision.generation);
|
||||
observation["transaction_id"] = serde_json::json!(revision.transaction_id);
|
||||
observation["pool_count"] = serde_json::json!(meta.pools.len());
|
||||
observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
|
||||
observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
|
||||
}
|
||||
PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
|
||||
PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
|
||||
PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
|
||||
PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
|
||||
}
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
}
|
||||
reads
|
||||
join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await
|
||||
}
|
||||
|
||||
fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
|
||||
@@ -5522,60 +5479,6 @@ fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
struct StartupCasObservation {
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: Vec<usize>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
tokio::task_local! {
|
||||
static STARTUP_CAS_OBSERVATION: StartupCasObservation;
|
||||
}
|
||||
|
||||
// This scope follows only the directly polled startup future. Spawned work
|
||||
// does not inherit it; receiver evidence retains its existing RPC tuple.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: &[Arc<S>],
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
STARTUP_CAS_OBSERVATION
|
||||
.scope(
|
||||
StartupCasObservation {
|
||||
attempt,
|
||||
phase,
|
||||
// These identities are never dereferenced or logged. The
|
||||
// caller and operation keep the same pool Arcs alive.
|
||||
pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
// Direct JSON diagnostics are independent of the startup tracing subscriber.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
|
||||
let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
|
||||
.ok()
|
||||
.and_then(|value| uuid::Uuid::parse_str(&value).ok())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
observation["nonce"] = serde_json::json!(nonce);
|
||||
observation["pid"] = serde_json::json!(std::process::id());
|
||||
let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
observation["attempt"] = serde_json::json!(scope.attempt);
|
||||
observation["startup_phase"] = serde_json::json!(scope.phase);
|
||||
});
|
||||
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
|
||||
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
|
||||
}
|
||||
|
||||
async fn save_pool_meta_object_cas<S>(
|
||||
pool: Arc<S>,
|
||||
object: &str,
|
||||
@@ -5596,43 +5499,13 @@ where
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_options(&mut opts);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
|
||||
serde_json::json!({
|
||||
"kind": "cas", "object": object, "phase": phase,
|
||||
"pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
|
||||
}).ok().flatten(),
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
|
||||
"if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
|
||||
"if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
|
||||
"tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
|
||||
"no_lock": opts.no_lock,
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let result = result.and_then(|object_info| {
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
});
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(mut observation) = observation {
|
||||
observation["ok"] = serde_json::json!(result.is_ok());
|
||||
observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
|
||||
observation["mod_time"] = serde_json::json!(
|
||||
result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|info| info.mod_time)
|
||||
.map(|time| time.unix_timestamp_nanos().to_string())
|
||||
);
|
||||
observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
result
|
||||
let object_info = result?;
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
}
|
||||
|
||||
async fn persist_pool_meta_identity<S>(
|
||||
@@ -6931,13 +6804,6 @@ impl PoolMeta {
|
||||
};
|
||||
if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
|
||||
persist_pool_meta_identity(pools, write_state, true, fence).await?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "confirmed", "object": POOL_META_NAME,
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
|
||||
"generation": confirmed.revision.generation,
|
||||
"transaction_id": confirmed.revision.transaction_id,
|
||||
}));
|
||||
return Ok(confirmed.meta);
|
||||
}
|
||||
if !commit_succeeded {
|
||||
@@ -19434,8 +19300,8 @@ mod pools_tests {
|
||||
load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
|
||||
merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
|
||||
missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
|
||||
parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
|
||||
read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica,
|
||||
reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
|
||||
renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
|
||||
reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
|
||||
@@ -20303,26 +20169,6 @@ mod pools_tests {
|
||||
assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_recovery_control_receipt_path_round_trips() {
|
||||
let run_token = "b".repeat(64);
|
||||
let control_id = "a".repeat(64);
|
||||
let source_path = format!(
|
||||
"ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
|
||||
&control_id[..2],
|
||||
&control_id[2..4],
|
||||
control_id
|
||||
);
|
||||
let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
|
||||
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
|
||||
|
||||
assert_eq!(locator.run_token, run_token);
|
||||
assert_eq!(locator.source_path, source_path);
|
||||
assert_eq!(locator.id_kind, "control_id");
|
||||
assert_eq!(locator.id, control_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_receipt_merge_preserves_terminal_proof() {
|
||||
let operation_id = "a".repeat(64);
|
||||
|
||||
@@ -99,17 +99,11 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
|
||||
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
|
||||
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
|
||||
|
||||
@@ -324,30 +324,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
// Preserve the old DiskAPI future's boxing boundary.
|
||||
Box::pin(
|
||||
self.disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -357,34 +333,6 @@ impl LocalDiskWrapper {
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::RenameDataObservation {
|
||||
self.rename_data_observed_with_guards(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
super::RenameDataGuards {
|
||||
external_guard,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_guards(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: super::RenameDataGuards,
|
||||
) -> super::RenameDataObservation {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
..
|
||||
} = guards;
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -409,15 +357,13 @@ impl LocalDiskWrapper {
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await;
|
||||
let observed =
|
||||
Box::pin(
|
||||
operation
|
||||
.disk
|
||||
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
|
||||
)
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
@@ -1355,7 +1301,6 @@ impl LocalDiskWrapper {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path_for_io(volume, path)
|
||||
}
|
||||
|
||||
@@ -218,12 +218,10 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
|
||||
+339
-1000
File diff suppressed because it is too large
Load Diff
@@ -17,14 +17,13 @@
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
use super::should_fail_local_inline_rollback_hardlink;
|
||||
use super::{
|
||||
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
|
||||
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
|
||||
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
|
||||
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
|
||||
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
|
||||
skip_access_checks,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
|
||||
@@ -34,7 +33,7 @@ use crate::disk::{
|
||||
error::{DiskError, Result},
|
||||
error_conv::{to_access_error, to_file_error},
|
||||
os,
|
||||
os::check_path_length,
|
||||
os::{check_path_length, rename_all},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
@@ -74,8 +73,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_rollback_path: Option<&Path>,
|
||||
) -> std::io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
|
||||
if let Some(backup_path) = local_rollback_path {
|
||||
// The commit immediately before this rollback renamed the staged
|
||||
// xl.meta from the same directory as `backup_path` onto
|
||||
@@ -89,7 +86,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
@@ -235,12 +231,6 @@ async fn restore_published_data_source(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
@@ -250,7 +240,7 @@ impl LocalDisk {
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
preflight_rejection: &mut Option<LocalRenamePreflightRejection>,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
let mut fi = fi;
|
||||
@@ -279,13 +269,7 @@ impl LocalDisk {
|
||||
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||
None => None,
|
||||
};
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -318,7 +302,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -336,7 +320,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -544,9 +528,7 @@ impl LocalDisk {
|
||||
// rename below.
|
||||
if fi_healing
|
||||
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = self
|
||||
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
|
||||
.await
|
||||
&& let Err(err) = self.move_to_trash(dst_data_path, true, false).await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
@@ -773,7 +755,7 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -811,7 +793,7 @@ impl LocalDisk {
|
||||
break;
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dir(dir).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -1042,15 +1024,7 @@ impl LocalDisk {
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
Some(mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1246,18 +1220,14 @@ impl LocalDisk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
};
|
||||
let mut preflight_rejection = None;
|
||||
let result = self
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state)
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection)
|
||||
.await;
|
||||
super::super::RenameDataObservation {
|
||||
result,
|
||||
preflight_rejection: state.preflight_rejection,
|
||||
preflight_rejection,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,14 +75,6 @@ use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
pub(crate) struct RenameDataObservation {
|
||||
pub(crate) result: Result<RenameDataResp>,
|
||||
@@ -198,17 +190,11 @@ pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path_resolve_stage: &'static str,
|
||||
pub(crate) metadata_lookup_stage: &'static str,
|
||||
pub(crate) metadata_validate_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_wait_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_task_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) file_open_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_map_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_copy_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) direct_read_copy_stage: &'static str,
|
||||
}
|
||||
|
||||
@@ -732,25 +718,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep local undo publication owned independently of the wrapper deadline.
|
||||
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
|
||||
pub(crate) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -770,12 +737,12 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
@@ -786,7 +753,7 @@ impl Disk {
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
@@ -955,7 +922,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io_if_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+49
-570
@@ -91,7 +91,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
|
||||
#[cfg(unix)]
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
@@ -151,7 +150,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
|
||||
@@ -164,7 +162,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
.lock()
|
||||
@@ -240,104 +237,11 @@ pub(crate) mod fsync_dir_recorder {
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub(crate) enum Stage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
type Key = (Stage, PathBuf);
|
||||
static BEFORE_PUBLICATION: LazyLock<Mutex<HashMap<Key, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) struct Guard(Key);
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
BEFORE_PUBLICATION.lock().remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
install_at(Stage::PreparedRename, path, hook)
|
||||
}
|
||||
|
||||
pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
let key = (stage, path.to_path_buf());
|
||||
assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none());
|
||||
Guard(key)
|
||||
}
|
||||
|
||||
pub(crate) fn run(stage: Stage, path: &Path) {
|
||||
let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf()));
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Controlled application-test pause at an existing physical executor boundary.
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub struct LocalPublicationPause {
|
||||
_hook: prepared_publication_test_hooks::Guard,
|
||||
entered: oneshot::Receiver<()>,
|
||||
_release: std::sync::mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
impl LocalPublicationPause {
|
||||
pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result<Self> {
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local(volume, path)
|
||||
.ok_or(DiskError::DiskNotFound)??;
|
||||
let stage = match stage {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
};
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
Ok(Self {
|
||||
_hook: hook,
|
||||
entered,
|
||||
_release: release,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
(&mut self.entered).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
pub(crate) mod windows_rename_test_hooks {
|
||||
use super::*;
|
||||
@@ -672,7 +576,6 @@ impl OpenedDstDirFsyncGroup {
|
||||
}
|
||||
|
||||
struct DstDirFsyncWaiter {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
|
||||
}
|
||||
|
||||
@@ -731,7 +634,6 @@ impl DstDirFsyncGroupCommit {
|
||||
fn enqueue_opened(
|
||||
&self,
|
||||
opened: OpenedDstDirFsyncGroup,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let mut registry = self.inner.lock();
|
||||
@@ -762,10 +664,7 @@ impl DstDirFsyncGroupCommit {
|
||||
group
|
||||
};
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.push_back(DstDirFsyncWaiter {
|
||||
result_tx,
|
||||
namespace_owner,
|
||||
});
|
||||
group_state.pending.push_back(DstDirFsyncWaiter { result_tx });
|
||||
let start_worker = !group_state.worker_running;
|
||||
if start_worker {
|
||||
group_state.worker_running = true;
|
||||
@@ -787,13 +686,7 @@ impl DstDirFsyncGroupCommit {
|
||||
fn remove_idle_group(&self, group: &Arc<DstDirFsyncGroup>) {
|
||||
let mut registry = self.inner.lock();
|
||||
let group_state = group.inner.lock();
|
||||
if !group_state.worker_running
|
||||
&& group_state.pending.is_empty()
|
||||
&& registry
|
||||
.groups
|
||||
.get(&group.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, group))
|
||||
{
|
||||
if !group_state.worker_running && group_state.pending.is_empty() {
|
||||
registry.groups.remove(&group.key);
|
||||
}
|
||||
}
|
||||
@@ -816,20 +709,16 @@ impl DstDirFsyncGroupCommit {
|
||||
&self,
|
||||
dir: &Path,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None)
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
let dir = group.dir.clone();
|
||||
let dir_file = group.dir_file.clone();
|
||||
fsync_spawn_blocking(move || {
|
||||
// The batch worker may be cancelled while this syscall is still running.
|
||||
let _namespace_owners = namespace_owners;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
|
||||
#[cfg(test)]
|
||||
{
|
||||
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
|
||||
@@ -844,118 +733,66 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Ve
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let _namespace_owners = namespace_owners;
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
fsync_dir(&group.dir).await
|
||||
}
|
||||
|
||||
struct DstDirFsyncWorkerGuard {
|
||||
group: Arc<DstDirFsyncGroup>,
|
||||
in_flight: usize,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl Drop for DstDirFsyncWorkerGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
async fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) {
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
// Cancellation must release queued owners, but the physical batch keeps
|
||||
// its own owners until its blocking syscall returns.
|
||||
let pending = {
|
||||
let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock();
|
||||
let mut group_state = self.group.inner.lock();
|
||||
let pending = std::mem::take(&mut group_state.pending);
|
||||
group_state.worker_running = false;
|
||||
if registry
|
||||
.groups
|
||||
.get(&self.group.key)
|
||||
.is_some_and(|group| Arc::ptr_eq(group, &self.group))
|
||||
{
|
||||
registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight);
|
||||
registry.groups.remove(&self.group.key);
|
||||
}
|
||||
pending
|
||||
};
|
||||
// Lease and channel destructors must run outside the registry locks.
|
||||
drop(pending);
|
||||
}
|
||||
}
|
||||
|
||||
fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) -> impl std::future::Future<Output = ()> {
|
||||
// Capture before spawning: shutdown may drop the future without polling it.
|
||||
let worker_guard = DstDirFsyncWorkerGuard {
|
||||
group: group.clone(),
|
||||
in_flight: 0,
|
||||
armed: true,
|
||||
};
|
||||
async move {
|
||||
let mut worker_guard = worker_guard;
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let mut batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
worker_guard.armed = false;
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let result = fsync_open_dst_dir_group(&group)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
worker_guard.in_flight = batch.len();
|
||||
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect();
|
||||
let result = fsync_open_dst_dir_group(&group, namespace_owners)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
worker_guard.in_flight = 0;
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
worker_guard.armed = false;
|
||||
group_state.worker_running = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
dir: impl AsRef<Path>,
|
||||
enabled: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
if !enabled {
|
||||
return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await;
|
||||
return fsync_dir(dir).await;
|
||||
}
|
||||
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir))
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?;
|
||||
if let Some(group) = worker {
|
||||
tokio::spawn(run_dst_dir_fsync_group_worker(group));
|
||||
}
|
||||
@@ -967,11 +804,8 @@ async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(
|
||||
dir: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
@@ -980,7 +814,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
admission: &FileSyncAdmission,
|
||||
) -> io::Result<()> {
|
||||
if dst_dir_fsync_group_commit_enabled() {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true).await
|
||||
} else {
|
||||
fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await
|
||||
}
|
||||
@@ -988,7 +822,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1395,8 +1229,6 @@ pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
// Independent of the quota claim; both survive cancellation of the waiter.
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
@@ -1406,18 +1238,10 @@ impl NamespaceMutationLease {
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
||||
acquire_namespace_mutation_lease_with_owner(path, None).await
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1427,15 +1251,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await;
|
||||
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
|
||||
@@ -1443,7 +1258,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
_namespace_guard: namespace_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1933,69 +1747,6 @@ pub async fn rename_all(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if namespace_owner.is_none() {
|
||||
return fsync_dir(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
fsync_spawn_blocking(move || {
|
||||
let _namespace_owner = namespace_owner;
|
||||
fsync_dir_std(path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = namespace_owner;
|
||||
fsync_dir(path).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_file_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_file(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
std::fs::remove_file(path)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_dir_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_dir(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_all_with_lease(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
@@ -2188,8 +1939,6 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
move || {
|
||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -2228,32 +1977,6 @@ pub async fn rename_all_ignore_missing_source(
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
src_file_path.to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
base_dir.as_ref().to_path_buf(),
|
||||
publication_root.clone(),
|
||||
false,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()),
|
||||
Err(err) => Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool {
|
||||
let Some(source_parent) = src_file_path.parent() else {
|
||||
@@ -2319,11 +2042,6 @@ async fn reliable_rename_inner_with_lease(
|
||||
let base_dir = base_dir.clone();
|
||||
move || {
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
}
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -6418,245 +6136,6 @@ mod tests {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let dir = temp_dir.path().canonicalize().expect("canonical fsync path");
|
||||
let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let first_owner = first_ctx.begin_namespace_commit();
|
||||
let second_owner = second_ctx.begin_namespace_commit();
|
||||
let first_probe = Arc::downgrade(&first_owner);
|
||||
let second_probe = Arc::downgrade(&second_owner);
|
||||
let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"),
|
||||
Some(first_owner),
|
||||
)
|
||||
.expect("queue first real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"),
|
||||
Some(second_owner),
|
||||
)
|
||||
.expect("queue second real waiter");
|
||||
assert!(second_worker.is_none(), "same directory must join the same batch");
|
||||
assert_eq!(group.inner.lock().pending.len(), 2);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook =
|
||||
prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone()));
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("batch must reach its physical fsync")
|
||||
.expect("physical fsync entry");
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]);
|
||||
assert!(
|
||||
group.inner.lock().pending.is_empty(),
|
||||
"both waiters were transferred into the physical batch"
|
||||
);
|
||||
let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let queued_owner = queued_ctx.begin_namespace_commit();
|
||||
let queued_probe = Arc::downgrade(&queued_owner);
|
||||
let queued_generation = queued_ctx.namespace_commit_generation();
|
||||
let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"),
|
||||
Some(queued_owner),
|
||||
)
|
||||
.expect("queue a waiter after the physical batch was frozen");
|
||||
assert!(queued_worker.is_none());
|
||||
assert_eq!(group.inner.lock().pending.len(), 1);
|
||||
drop((first_rx, second_rx));
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled());
|
||||
assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation");
|
||||
assert!(queued_probe.upgrade().is_none());
|
||||
assert!(!queued_ctx.namespace_commits_pending());
|
||||
assert!(queued_ctx.namespace_commit_generation() > queued_generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some();
|
||||
let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some();
|
||||
let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation());
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while Arc::strong_count(&group.dir_file) != 1
|
||||
|| first_probe.upgrade().is_some()
|
||||
|| second_probe.upgrade().is_some()
|
||||
|| first_ctx.namespace_commits_pending()
|
||||
|| second_ctx.namespace_commits_pending()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical fsync must release every batch owner");
|
||||
assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute");
|
||||
assert!(
|
||||
first_pending && second_pending,
|
||||
"one physical batch must preserve both independent namespace owners"
|
||||
);
|
||||
assert!(!first_ctx.namespace_commits_pending());
|
||||
assert!(!second_ctx.namespace_commits_pending());
|
||||
assert!(first_ctx.namespace_commit_generation() > generations.0);
|
||||
assert!(second_ctx.namespace_commit_generation() > generations.1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_unpolled_worker_releases_queued_owner() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("queue a real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let worker = run_dst_dir_fsync_group_worker(group.clone());
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
drop(worker);
|
||||
assert!(rx.await.is_err(), "shutdown before first poll must release the waiter");
|
||||
assert!(probe.upgrade().is_none());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(ctx.namespace_commit_generation() > generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
assert!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(),
|
||||
"the dropped future must not dispatch a physical batch"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn stale_idle_group_cleanup_preserves_successor_registration() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let registry = DstDirFsyncGroupCommit::default();
|
||||
let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker");
|
||||
let old_group = first_worker.expect("first waiter starts a worker");
|
||||
// W1 has completed its batch and marked G idle, but has not cleaned G up.
|
||||
let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter");
|
||||
registry.complete_batch(1);
|
||||
old_group.inner.lock().worker_running = false;
|
||||
|
||||
let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker");
|
||||
let reused_group = second_worker.expect("idle G starts another worker");
|
||||
assert!(Arc::ptr_eq(&old_group, &reused_group));
|
||||
let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter");
|
||||
registry.complete_batch(1);
|
||||
reused_group.inner.lock().worker_running = false;
|
||||
registry.remove_idle_group(&reused_group);
|
||||
assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G");
|
||||
assert!(second_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(second_rx.try_recv().expect("second worker reports completion").is_ok());
|
||||
|
||||
let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
// W1 resumes with its old Arc after W2 removed G and W3 installed G2.
|
||||
registry.remove_idle_group(&old_group);
|
||||
assert!(first_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(first_rx.try_recv().expect("first worker reports completion").is_ok());
|
||||
assert!(
|
||||
registry
|
||||
.inner
|
||||
.lock()
|
||||
.groups
|
||||
.get(&successor.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, &successor)),
|
||||
"stale cleanup must retain the exact successor Arc"
|
||||
);
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
assert!(successor.inner.lock().worker_running);
|
||||
assert_eq!(successor.inner.lock().pending.len(), 1);
|
||||
assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)));
|
||||
|
||||
let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor");
|
||||
assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3");
|
||||
assert_eq!(successor.inner.lock().pending.len(), 2);
|
||||
assert_eq!(registry.counts_for_test(), (1, 2));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_for_test(temp_dir.path())
|
||||
.expect("enqueue old group");
|
||||
let old_group = old_worker.expect("old group starts a worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone()))
|
||||
.await
|
||||
.expect("old worker must finish its actual fsync");
|
||||
assert!(old_rx.await.expect("old worker reports completion").is_ok());
|
||||
assert!(fsync_dir_recorder::was_fsynced(temp_dir.path()));
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("enqueue successor owner");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
let worker = run_dst_dir_fsync_group_worker(successor.clone());
|
||||
// The stale Arc represents W1 resuming after another worker removed G.
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(probe.upgrade().is_some());
|
||||
drop(worker);
|
||||
let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx)
|
||||
.await
|
||||
.expect("dropping the unpolled worker must release its channel")
|
||||
.is_err();
|
||||
let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test();
|
||||
let owner_released = probe.upgrade().is_none();
|
||||
let namespace_pending = ctx.namespace_commits_pending();
|
||||
let generation_after_drop = ctx.namespace_commit_generation();
|
||||
let successor_pending = successor.inner.lock().pending.len();
|
||||
let worker_running = successor.inner.lock().worker_running;
|
||||
// Preserve the observed result before cleanup, so a RED run cannot leak
|
||||
// its phantom count into unrelated tests in the same process.
|
||||
clear_dst_dir_fsync_group_commit_for_test();
|
||||
assert!(channel_closed);
|
||||
assert!(owner_released);
|
||||
assert!(!namespace_pending);
|
||||
assert!(generation_after_drop > generation);
|
||||
assert_eq!(successor_pending, 0);
|
||||
assert!(!worker_running);
|
||||
assert_eq!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()),
|
||||
vec![1],
|
||||
"dropping the successor before its first poll must not dispatch another fsync"
|
||||
);
|
||||
assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() {
|
||||
|
||||
@@ -13,15 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
@@ -409,17 +406,11 @@ async fn open_disk_reader(
|
||||
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
|
||||
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
|
||||
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
#[cfg(unix)]
|
||||
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
#[cfg(unix)]
|
||||
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
|
||||
#[cfg(unix)]
|
||||
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
#[cfg(unix)]
|
||||
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
|
||||
#[cfg(unix)]
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
#[cfg(unix)]
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
let mmap_result = {
|
||||
|
||||
@@ -72,29 +72,6 @@ const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
|
||||
fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
|
||||
configured
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.unwrap_or(rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS)
|
||||
.min(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
fn admin_peer_probe_timeout() -> Duration {
|
||||
let configured = rustfs_utils::get_env_opt_u64_with_aliases(rustfs_config::ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, &[]);
|
||||
let seconds = resolve_admin_peer_probe_timeout_secs(configured);
|
||||
Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn remaining_admin_peer_probe_timeout(deadline: Instant) -> Option<Duration> {
|
||||
remaining_admin_peer_probe_timeout_at(deadline, Instant::now())
|
||||
}
|
||||
|
||||
fn remaining_admin_peer_probe_timeout_at(deadline: Instant, now: Instant) -> Option<Duration> {
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
(!remaining.is_zero()).then_some(remaining)
|
||||
}
|
||||
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
@@ -1561,7 +1538,7 @@ impl NotificationSys {
|
||||
{
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
|
||||
let peer_timeout = admin_peer_probe_timeout();
|
||||
let peer_timeout = Duration::from_secs(5);
|
||||
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
let endpoints = endpoints.clone();
|
||||
@@ -1569,9 +1546,7 @@ impl NotificationSys {
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
let host = client.host.to_string();
|
||||
let deadline = Instant::now() + peer_timeout;
|
||||
let probe_timeout = remaining_admin_peer_probe_timeout(deadline).unwrap_or_default();
|
||||
match timeout(probe_timeout, client.local_storage_info()).await {
|
||||
match timeout(peer_timeout, client.local_storage_info()).await {
|
||||
Ok(Ok(mut info)) => {
|
||||
normalize_and_cache_peer_storage_info(cache, &host, &mut info);
|
||||
Some(info)
|
||||
@@ -1582,6 +1557,7 @@ impl NotificationSys {
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
|
||||
client.evict_connection().await;
|
||||
handle_peer_failure(cache, &host, &endpoints)
|
||||
}
|
||||
}
|
||||
@@ -1607,7 +1583,7 @@ impl NotificationSys {
|
||||
pub async fn server_info(&self) -> Vec<ServerProperties> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
|
||||
let peer_timeout = admin_peer_probe_timeout();
|
||||
let peer_timeout = Duration::from_secs(5);
|
||||
|
||||
for (idx, client) in self.peer_clients.iter().enumerate() {
|
||||
let host = self
|
||||
@@ -1624,23 +1600,12 @@ impl NotificationSys {
|
||||
};
|
||||
};
|
||||
|
||||
let deadline = Instant::now() + peer_timeout;
|
||||
let Some(first_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
|
||||
// First attempt. A single evicted or half-open internode channel
|
||||
// is enough to fail one probe and, before retrying, would drop
|
||||
// the member to unknown/offline for this whole snapshot. On a
|
||||
// quick failure we evict the channel and re-dial once before
|
||||
// falling back (rustfs/backlog#1049, P1-B). A slow attempt
|
||||
// consumes the round budget and therefore does not trigger a
|
||||
// second full wait or an asynchronous eviction side effect.
|
||||
match timeout(first_timeout, client.server_info()).await {
|
||||
// the member to unknown/offline for this whole snapshot. So on any
|
||||
// first-attempt failure we evict the channel and re-dial once
|
||||
// before falling back (rustfs/backlog#1049, P1-B).
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => {
|
||||
return PeerServerInfoProbe { host, result: Ok(info) };
|
||||
}
|
||||
@@ -1654,37 +1619,14 @@ impl NotificationSys {
|
||||
// `evict_connection` would leave that gate up and the retry would
|
||||
// fast-fail with "temporarily offline" instead of reconnecting
|
||||
// (rustfs/backlog#1049 P1-B).
|
||||
let Some(retry_budget) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
// Bound connection-cache cleanup too. The helper clears the offline gate even
|
||||
// when eviction itself times out, so cancellation cannot strand this peer in
|
||||
// fast-fail mode.
|
||||
if !client.prepare_retry_with_timeout(retry_budget).await {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
}
|
||||
client.prepare_retry().await;
|
||||
|
||||
// Second and final attempt on the fresh channel.
|
||||
let Some(retry_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
return PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
};
|
||||
};
|
||||
match timeout(retry_timeout, client.server_info()).await {
|
||||
match timeout(peer_timeout, client.server_info()).await {
|
||||
Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
|
||||
Ok(Err(err)) => {
|
||||
warn!("peer {host} server_info failed after retry: {err}");
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
let health = peer_disk_health(&host).await;
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
@@ -1692,7 +1634,8 @@ impl NotificationSys {
|
||||
}
|
||||
Err(_) => {
|
||||
warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
|
||||
let health = peer_disk_health_with_deadline(&host, deadline).await;
|
||||
client.evict_connection().await;
|
||||
let health = peer_disk_health(&host).await;
|
||||
PeerServerInfoProbe {
|
||||
host,
|
||||
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
|
||||
@@ -3080,11 +3023,6 @@ async fn peer_disk_health(host: &str) -> Option<PeerDiskHealth> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn peer_disk_health_with_deadline(host: &str, deadline: Instant) -> Option<PeerDiskHealth> {
|
||||
let remaining = remaining_admin_peer_probe_timeout(deadline)?;
|
||||
timeout(remaining, peer_disk_health(host)).await.ok().flatten()
|
||||
}
|
||||
|
||||
/// Handle a peer failure for server_info: return cached data if available, or
|
||||
/// classify the member as `unknown` / `degraded` / `offline` depending on how
|
||||
/// many consecutive probes have failed and whether the peer's drives are still
|
||||
@@ -4079,37 +4017,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_peer_probe_timeout_rejects_zero_and_caps_large_values() {
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(None),
|
||||
rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(Some(0)),
|
||||
rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_admin_peer_probe_timeout_secs(Some(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + 1)),
|
||||
rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS
|
||||
);
|
||||
assert_eq!(resolve_admin_peer_probe_timeout_secs(Some(7)), 7);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_peer_probe_health_fallback_respects_expired_deadline() {
|
||||
let deadline = Instant::now();
|
||||
assert!(peer_disk_health_with_deadline("peer-1", deadline).await.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_peer_probe_deadline_is_shared_across_attempts() {
|
||||
let start = Instant::now();
|
||||
let deadline = start + Duration::from_secs(10);
|
||||
assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(6)).is_some());
|
||||
assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(10)).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn call_peer_with_timeout_returns_value_when_fast() {
|
||||
let result = call_peer_with_timeout(
|
||||
|
||||
@@ -56,7 +56,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
#[cfg(feature = "test-util")]
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -69,28 +68,21 @@ use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::format::FormatV3;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the barrier for the next rejected transition cleanup.
|
||||
pub fn install() -> Self {
|
||||
@@ -104,7 +96,6 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
|
||||
/// Default polling cadence used by the `wait_for_*` helpers.
|
||||
#[cfg(feature = "test-util")]
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// A fault to inject into [`MockWarmBackend`] operations.
|
||||
@@ -217,12 +208,10 @@ impl Drop for MockRemoveOperationGuard {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockPutBarrier {
|
||||
/// Wait until the remote body is stored and the PUT is paused before returning.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -237,7 +226,6 @@ impl MockPutBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockPutBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -270,12 +258,10 @@ impl Drop for MockGetBarrier {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses and then fails a mock tier DELETE.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockRemoveBarrier {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockRemoveBarrier {
|
||||
/// Wait until DELETE reaches the deterministic failure point.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -297,7 +283,6 @@ impl MockRemoveBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockRemoveBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -321,7 +306,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause after the next tier PUT stores its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
|
||||
let state = Arc::new(MockPutBarrierState::default());
|
||||
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
|
||||
@@ -329,7 +313,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Pause and then fail the next DELETE after it reaches the backend.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
|
||||
let state = Arc::new(MockRemoveBarrierState::default());
|
||||
let mut barrier = self.inner.remove_barrier.lock().await;
|
||||
@@ -340,7 +323,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then return an error
|
||||
/// after the test releases it.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
|
||||
let state = Arc::new(MockGetBarrierState {
|
||||
fail_after_release: true,
|
||||
@@ -361,7 +343,6 @@ impl MockWarmBackend {
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_faults(&self, faults: FaultConfig) {
|
||||
*self.inner.faults.lock().await = faults;
|
||||
}
|
||||
@@ -372,7 +353,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Toggle "HTTP 5xx" server errors on every operation.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_server_error(&self, server_error: bool) {
|
||||
self.inner.faults.lock().await.server_error = server_error;
|
||||
}
|
||||
@@ -383,13 +363,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Set (or clear, with `None`) injected latency applied before each op.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_latency(&self, latency: Option<Duration>) {
|
||||
self.inner.faults.lock().await.latency = latency;
|
||||
}
|
||||
|
||||
/// Clear all injected faults, restoring healthy behaviour.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_faults(&self) {
|
||||
*self.inner.faults.lock().await = FaultConfig::default();
|
||||
}
|
||||
@@ -397,7 +375,6 @@ impl MockWarmBackend {
|
||||
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
|
||||
/// the complete body. This models a backend that incorrectly accepts a
|
||||
/// truncated stream while still returning success.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
|
||||
*self.inner.put_read_limit.lock().await = limit;
|
||||
}
|
||||
@@ -418,14 +395,12 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Reject the next non-empty remote version validation without changing
|
||||
/// subsequent exact-version backend cleanup behavior.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn reject_next_non_empty_remote_version_validation(&self) {
|
||||
self.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
@@ -463,7 +438,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Clear the operation log without touching stored objects or faults.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_op_log(&self) {
|
||||
self.inner.op_log.lock().await.clear();
|
||||
}
|
||||
@@ -485,13 +459,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs passed to successful tier removes.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn remove_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
@@ -503,7 +475,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn get_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -515,7 +486,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Number of `put` calls recorded.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -529,7 +499,6 @@ impl MockWarmBackend {
|
||||
// ---- storage inspection ---------------------------------------------
|
||||
|
||||
/// Whether the backend currently stores `object`.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn contains(&self, object: &str) -> bool {
|
||||
self.inner.objects.lock().await.contains_key(object)
|
||||
}
|
||||
@@ -540,13 +509,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// A clone of the stored object, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
|
||||
self.inner.objects.lock().await.get(object).cloned()
|
||||
}
|
||||
|
||||
/// A clone of the raw bytes stored for `object`, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
|
||||
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
|
||||
}
|
||||
@@ -571,7 +538,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until `object` is absent from the backend, or `timeout` elapses.
|
||||
/// Returns `true` if the object disappeared within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -587,7 +553,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until the backend holds exactly `expected` objects, or `timeout`
|
||||
/// elapses. Returns `true` if the count was reached within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -882,7 +847,6 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
|
||||
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
|
||||
/// free-version count.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionMeta {
|
||||
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
|
||||
pub status: String,
|
||||
@@ -896,7 +860,6 @@ pub struct TransitionMeta {
|
||||
pub free_version_count: usize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
|
||||
// with the position recorded in the disk's own format.json, so derive the
|
||||
@@ -927,7 +890,6 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
/// The free-version metadata removal lands asynchronously after the remote
|
||||
/// object disappears, so callers typically poll via
|
||||
/// [`wait_for_free_version_absence`] instead of asserting a single read.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
|
||||
let Some(disk) = open_disk(disk_path).await else {
|
||||
return 0;
|
||||
@@ -952,7 +914,6 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
|
||||
/// fields are taken from the newest version that carries a transition record;
|
||||
/// if no version is transitioned, they are taken from the current version (and
|
||||
/// will be empty).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
|
||||
let disk = open_disk(disk_path).await?;
|
||||
let data = disk
|
||||
@@ -986,7 +947,6 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
/// disk is missing the object or disagrees — this is the shard-consistency
|
||||
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
|
||||
/// four-tuple plus free-version count must match across all erasure shards).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
|
||||
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
|
||||
|
||||
@@ -1012,7 +972,6 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
|
||||
|
||||
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
|
||||
/// elapses. Returns `true` if the free versions drained within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -1081,44 +1040,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
|
||||
let backend = MockWarmBackend::new();
|
||||
let metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("project".to_string(), "archive".to_string()),
|
||||
]);
|
||||
let version = backend
|
||||
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
|
||||
.await
|
||||
.expect("mock PUT should preserve remote metadata");
|
||||
assert_eq!(backend.metadata("object").await, Some(metadata));
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe stored object"),
|
||||
TransitionCandidateProbe::VersionedPresent(version)
|
||||
);
|
||||
|
||||
backend.external_remove("object").await;
|
||||
assert_eq!(backend.metadata("object").await, None);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe removed object"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
let operations = backend.op_log().await;
|
||||
assert!(
|
||||
operations
|
||||
.iter()
|
||||
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
|
||||
);
|
||||
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_preserves_fault_fail_closed_behavior() {
|
||||
let backend = MockWarmBackend::new();
|
||||
|
||||
@@ -143,12 +143,11 @@ struct TierDriverBuildBarrier {
|
||||
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type TierDriverTestFactory =
|
||||
Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
|
||||
type TierDriverTestFactory = Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(crate) static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
|
||||
static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1378,42 +1377,27 @@ async fn ensure_no_authoritative_persisted_references<S>(
|
||||
where
|
||||
S: TierReferenceProofStore,
|
||||
{
|
||||
ensure_no_authoritative_persisted_references_with(
|
||||
api.clone(),
|
||||
TIER_DELETE_JOURNAL_PREFIX,
|
||||
"tier-delete journal",
|
||||
|_object, data| {
|
||||
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
journal.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
|
||||
))
|
||||
},
|
||||
)
|
||||
ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| {
|
||||
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
journal.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
|
||||
))
|
||||
})
|
||||
.await?;
|
||||
ensure_no_authoritative_persisted_references_with(
|
||||
api,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX,
|
||||
"transition transaction",
|
||||
|object, data| {
|
||||
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
transaction.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(
|
||||
&transaction.tier_name,
|
||||
Some(transaction.backend_fingerprint),
|
||||
targets,
|
||||
),
|
||||
))
|
||||
},
|
||||
)
|
||||
ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| {
|
||||
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
transaction.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets),
|
||||
))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn ensure_no_authoritative_persisted_references_with<S, F>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
reference_kind: &str,
|
||||
blocks_target: F,
|
||||
) -> std::result::Result<(), AdminError>
|
||||
where
|
||||
@@ -1442,7 +1426,7 @@ where
|
||||
.map_err(tier_reference_proof_admin_error)?;
|
||||
let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?;
|
||||
if blocks {
|
||||
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, reference_kind, &object.name));
|
||||
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name));
|
||||
}
|
||||
}
|
||||
if !page.is_truncated {
|
||||
@@ -1504,21 +1488,16 @@ fn tier_persisted_reference_blocks_target(
|
||||
|
||||
fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
let reference_kind = if object.transitioned_object.free_version {
|
||||
"free-version ownership"
|
||||
} else {
|
||||
"transitioned-object"
|
||||
};
|
||||
err.message = format!(
|
||||
"Remote tier {tier_name} still has a {reference_kind} reference, for example {}/{}",
|
||||
"Remote tier {tier_name} still has object references, for example {}/{}",
|
||||
object.bucket, object.name
|
||||
);
|
||||
err
|
||||
}
|
||||
|
||||
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, reference_kind: &str, object: &str) -> AdminError {
|
||||
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
err.message = format!("Remote tier {tier_name} still has a {reference_kind} reference, for example {object}");
|
||||
err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}");
|
||||
err
|
||||
}
|
||||
|
||||
@@ -3706,17 +3685,13 @@ impl TierConfigMgr {
|
||||
let manager = handle.read().await;
|
||||
let runtime = tier_driver_runtime(handle, &manager);
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
let prepared = runtime
|
||||
.prepared_mutation_blocks
|
||||
.values()
|
||||
.any(|blocked_mutation_id| *blocked_mutation_id == mutation_id);
|
||||
let committed = runtime
|
||||
if !runtime
|
||||
.committed_mutation_blocks
|
||||
.values()
|
||||
.any(|mutation_ids| mutation_ids.contains(&mutation_id));
|
||||
if !prepared && !committed {
|
||||
.any(|mutation_ids| mutation_ids.contains(&mutation_id))
|
||||
{
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation fence was not installed".to_string();
|
||||
err.message = "Remote tier committed mutation fence was not installed".to_string();
|
||||
return Err(err);
|
||||
}
|
||||
Ok(MutationBlockAllowance {
|
||||
@@ -3923,6 +3898,14 @@ impl TierConfigMgr {
|
||||
Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, mutation_block_allowance)
|
||||
}
|
||||
|
||||
fn begin_tier_transition(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
manager: &mut Self,
|
||||
changed: HashSet<String>,
|
||||
) -> std::result::Result<TierPublishTransition, AdminError> {
|
||||
Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None)
|
||||
}
|
||||
|
||||
fn begin_tier_transition_with_destinations(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
manager: &mut Self,
|
||||
@@ -4193,7 +4176,7 @@ impl TierConfigMgr {
|
||||
let mut config_lock = config_lock;
|
||||
let coordinated_config_update = config_lock.is_some();
|
||||
let mut update = Some(update);
|
||||
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, target_tiers) =
|
||||
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) =
|
||||
match mutation {
|
||||
TierCandidateMutation::Prevalidated(prepared) => {
|
||||
if version != prepared.version {
|
||||
@@ -4209,8 +4192,9 @@ impl TierConfigMgr {
|
||||
)));
|
||||
}
|
||||
candidate = prepared.candidate;
|
||||
let target_tiers = {
|
||||
let manager = handle.read().await;
|
||||
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
let mut transition = {
|
||||
let mut manager = handle.write().await;
|
||||
let mut target_tiers = changed_tier_names(&manager, &candidate);
|
||||
if let Some(tier_name) = prepared.explicit_tier_name.as_ref()
|
||||
&& (manager.tiers.contains_key(tier_name)
|
||||
@@ -4219,7 +4203,8 @@ impl TierConfigMgr {
|
||||
{
|
||||
target_tiers.insert(tier_name.clone());
|
||||
}
|
||||
target_tiers
|
||||
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
(
|
||||
prepared.kind,
|
||||
@@ -4227,7 +4212,7 @@ impl TierConfigMgr {
|
||||
prepared.force,
|
||||
prepared.current,
|
||||
prepared.driver_tier,
|
||||
target_tiers,
|
||||
transition,
|
||||
)
|
||||
}
|
||||
mutation => {
|
||||
@@ -4252,9 +4237,11 @@ impl TierConfigMgr {
|
||||
last_refreshed_at: candidate.last_refreshed_at,
|
||||
};
|
||||
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
let target_tiers = {
|
||||
let manager = handle.read().await;
|
||||
mutation.target_tiers(&manager, &candidate)
|
||||
let mut transition = {
|
||||
let mut manager = handle.write().await;
|
||||
let target_tiers = mutation.target_tiers(&manager, &candidate);
|
||||
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline)
|
||||
.await
|
||||
@@ -4265,7 +4252,7 @@ impl TierConfigMgr {
|
||||
mutation_force,
|
||||
current_for_targets,
|
||||
driver_tier,
|
||||
target_tiers,
|
||||
transition,
|
||||
)
|
||||
}
|
||||
};
|
||||
@@ -4288,83 +4275,28 @@ impl TierConfigMgr {
|
||||
save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref())
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Save)?;
|
||||
let mut blocked_target_tiers = target_tiers.clone();
|
||||
if let Some(intent) = coordinator_intent.as_ref() {
|
||||
blocked_target_tiers.extend(intent.affected_targets.iter().map(|target| target.tier_name.clone()));
|
||||
}
|
||||
if let Some(intent) = coordinator_intent.as_ref() {
|
||||
// `target_tiers` may include a stale local-only manager
|
||||
// entry that is absent from the persisted proof
|
||||
// snapshot. Fence that local transition under the same
|
||||
// mutation ID as well; recovery may discard this
|
||||
// process-local superset, which advances the revision
|
||||
// and makes the deferred transition fail closed.
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block_for_tiers(&handle, intent, &blocked_target_tiers)
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&handle, intent)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?;
|
||||
}
|
||||
let prepared_mutation_block_allowance = match coordinator_intent.as_ref() {
|
||||
Some(intent) => Some(
|
||||
TierConfigMgr::mutation_block_allowance_for(&handle, intent.mutation_id)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
// A durable coordinator intent supplies the admission
|
||||
// fence that lets us defer generation revocation. Keep the
|
||||
// original early transition for no-intent paths (for
|
||||
// example, reconciling a stale local manager to an
|
||||
// idempotently removed persisted tier), where there is no
|
||||
// Prepared record capable of blocking a new lease.
|
||||
let (mut transition, deferred_target_tiers) = if coordinator_intent.is_some() {
|
||||
(None, Some(target_tiers))
|
||||
} else {
|
||||
let transition = {
|
||||
let mut manager = handle.write().await;
|
||||
Self::begin_tier_transition_with_destinations(
|
||||
&handle,
|
||||
&mut manager,
|
||||
target_tiers,
|
||||
HashMap::new(),
|
||||
None,
|
||||
)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
(Some(transition), None)
|
||||
};
|
||||
if coordinated_config_update {
|
||||
drop(update.take());
|
||||
drop(config_lock.take());
|
||||
}
|
||||
// The durable Prepared block closes admission before the
|
||||
// zero-reference proof, but deliberately leaves already
|
||||
// issued generations current. In particular, an exact
|
||||
// free-version cleanup that has completed remote DELETE
|
||||
// must still be able to remove its local ownership marker;
|
||||
// revoking its generation here would strand that marker
|
||||
// and make this mutation reject its own interrupted work.
|
||||
if let Some(intent) = coordinator_intent.as_ref()
|
||||
&& let Err(drain_error) =
|
||||
TierConfigMgr::wait_for_blocked_tier_operation_leases_for_tiers(&handle, &blocked_target_tiers).await
|
||||
{
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), Some(intent), Vec::new()).await {
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await {
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
mutation_id = %intent.mutation_id,
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation lease drain failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(drain_error));
|
||||
} else if let Some(transition) = transition.as_ref() {
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
transition
|
||||
.wait_for_active_leases_until(drain_deadline)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?;
|
||||
}
|
||||
let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() {
|
||||
let peers = match remote_tier_mutation_peers().await {
|
||||
@@ -4431,71 +4363,6 @@ impl TierConfigMgr {
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(proof_error));
|
||||
}
|
||||
// No affected-tier lease can start after Prepared, and the
|
||||
// existing set was drained above. It is now safe to revoke
|
||||
// the generation for publication without invalidating a
|
||||
// cleanup between its remote and local commit boundaries.
|
||||
if transition.is_none() {
|
||||
let target_tiers = deferred_target_tiers.ok_or_else(|| {
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation lost its deferred transition targets".to_string();
|
||||
TierConfigUpdateError::Publish(err)
|
||||
})?;
|
||||
let mut manager = handle.write().await;
|
||||
transition = Some(
|
||||
match Self::begin_tier_transition_with_destinations(
|
||||
&handle,
|
||||
&mut manager,
|
||||
target_tiers,
|
||||
HashMap::new(),
|
||||
prepared_mutation_block_allowance.as_ref(),
|
||||
) {
|
||||
Ok(transition) => transition,
|
||||
Err(transition_error) => {
|
||||
drop(manager);
|
||||
if !abort_prepared_tier_mutation(
|
||||
&handle,
|
||||
api.clone(),
|
||||
coordinator_intent.as_ref(),
|
||||
prepared_peers,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation publish transition failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(transition_error));
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
let mut transition = transition.ok_or_else(|| {
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation lost its publish transition".to_string();
|
||||
TierConfigUpdateError::Publish(err)
|
||||
})?;
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
|
||||
drop(transition);
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await
|
||||
{
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation publish drain failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(drain_error));
|
||||
}
|
||||
let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?;
|
||||
if coordinated_config_update {
|
||||
config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await {
|
||||
@@ -5547,40 +5414,11 @@ impl TierConfigMgr {
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let target_tiers = intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.map(|target| target.tier_name.clone())
|
||||
.collect();
|
||||
Self::apply_prepared_mutation_intent_block_for_tiers(handle, intent, &target_tiers).await
|
||||
}
|
||||
|
||||
async fn apply_prepared_mutation_intent_block_for_tiers(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
target_tiers: &HashSet<String>,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
if intent.state != TierMutationIntentState::Prepared {
|
||||
return Ok(());
|
||||
}
|
||||
let manager = handle.read().await;
|
||||
let runtime = tier_driver_runtime(handle, &manager);
|
||||
let mut runtime = lock_unpoisoned(&runtime);
|
||||
let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
|
||||
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
|
||||
for tier_name in target_tiers {
|
||||
match prepared_mutation_blocks.entry(tier_name.clone()) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(intent.mutation_id);
|
||||
}
|
||||
Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
|
||||
Entry::Occupied(_) => {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
err.message = format!("Remote tier {tier_name} already has another prepared mutation");
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
if prepared_mutation_blocks == runtime.prepared_mutation_blocks {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -5596,18 +5434,6 @@ impl TierConfigMgr {
|
||||
pub(crate) async fn wait_for_blocked_tier_operation_leases(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let target_tiers = intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.map(|target| target.tier_name.clone())
|
||||
.collect();
|
||||
Self::wait_for_blocked_tier_operation_leases_for_tiers(handle, &target_tiers).await
|
||||
}
|
||||
|
||||
async fn wait_for_blocked_tier_operation_leases_for_tiers(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
target_tiers: &HashSet<String>,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let generations = {
|
||||
let manager = handle.read().await;
|
||||
@@ -5615,9 +5441,10 @@ impl TierConfigMgr {
|
||||
return Ok(());
|
||||
};
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
target_tiers
|
||||
intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.filter_map(|tier_name| runtime.generations.get(tier_name).cloned())
|
||||
.filter_map(|target| runtime.generations.get(&target.tier_name).cloned())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let drain = async {
|
||||
@@ -14587,13 +14414,6 @@ mod tests {
|
||||
.push(object);
|
||||
}
|
||||
|
||||
fn remove_listed_version(&self, bucket: &str, object: &str) {
|
||||
self.listed_versions
|
||||
.lock()
|
||||
.expect("tier reference fixture should not poison")
|
||||
.retain(|version| version.bucket != bucket || version.name != object);
|
||||
}
|
||||
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
|
||||
self.lifecycle_configs
|
||||
.lock()
|
||||
@@ -15862,152 +15682,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_remove_prepared_fence_allows_inflight_free_version_cleanup_to_finish() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let tier = build_rustfs_tier("COLD-A");
|
||||
let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("free-version drain fixture should persist");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
guard.tiers.insert("COLD-A".to_string(), tier);
|
||||
guard.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
|
||||
guard
|
||||
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("cleanup")))
|
||||
.expect("cleanup driver generation should install");
|
||||
guard
|
||||
.replace_driver("COLD-B", Box::new(LeaseTestBackend::ready("stale-local")))
|
||||
.expect("stale local driver generation should install");
|
||||
}
|
||||
let cleanup_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("in-flight cleanup lease should be available");
|
||||
let stale_local_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
|
||||
.await
|
||||
.expect("stale local tier lease should be available");
|
||||
let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(identity));
|
||||
free_version.transitioned_object.status = "pending".to_string();
|
||||
free_version.transitioned_object.free_version = true;
|
||||
store.add_listed_version(free_version);
|
||||
|
||||
let remove_manager = manager.clone();
|
||||
let remove_store = store.clone();
|
||||
let remove = tokio::spawn(async move {
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = {
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
runtime.prepared_mutation_blocks.contains_key("COLD-A")
|
||||
&& runtime.prepared_mutation_blocks.contains_key("COLD-B")
|
||||
};
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("tier remove should install its durable prepared fence");
|
||||
|
||||
assert!(
|
||||
cleanup_lease.is_current(&manager).await,
|
||||
"the prepared fence must let the already leased cleanup finish its exact local marker deletion"
|
||||
);
|
||||
assert!(
|
||||
stale_local_lease.is_current(&manager).await,
|
||||
"the local superset fence must also let an already leased stale-manager operation finish"
|
||||
);
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("the prepared fence must reject new tier operations"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
|
||||
let stale_blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-B").await {
|
||||
Ok(_) => panic!("the local superset fence must reject new stale-manager operations"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&stale_blocked));
|
||||
|
||||
store.remove_listed_version("photos", "2026/free-version.jpg");
|
||||
drop(cleanup_lease);
|
||||
drop(stale_local_lease);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), remove)
|
||||
.await
|
||||
.expect("tier remove should finish after both in-flight operations release their leases")
|
||||
.expect("tier remove task should join")
|
||||
.expect("tier remove should pass once the in-flight cleanup removes its marker");
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-B"));
|
||||
assert!(
|
||||
!load_tier_config_for_update(store)
|
||||
.await
|
||||
.expect("removed tier config should reload")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_intent_stale_manager_removal_keeps_early_generation_drain() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
empty_mgr()
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("empty persisted tier config should exist");
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("stale"));
|
||||
}
|
||||
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("stale manager lease should be available");
|
||||
|
||||
let remove_manager = manager.clone();
|
||||
let remove_store = store.clone();
|
||||
let remove =
|
||||
tokio::spawn(async move { TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true).await });
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("no-intent stale-manager reconciliation should revoke before its proof");
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("stale-manager reconciliation must not admit a new operation"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
|
||||
|
||||
drop(old);
|
||||
remove
|
||||
.await
|
||||
.expect("stale-manager removal task should join")
|
||||
.expect("stale-manager removal should converge to the persisted empty config");
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
|
||||
}
|
||||
|
||||
async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let tier = build_rustfs_tier("COLD-A");
|
||||
@@ -16911,23 +16585,12 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned update should install its prepared fence before caller cancellation");
|
||||
assert!(
|
||||
old.is_current(&manager).await,
|
||||
"an already leased operation must remain current until it can finish"
|
||||
);
|
||||
.expect("owned update should revoke before caller cancellation");
|
||||
caller.abort();
|
||||
drop(old);
|
||||
|
||||
@@ -16972,23 +16635,12 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned update should install its prepared fence before caller cancellation");
|
||||
assert!(
|
||||
old.is_current(&manager).await,
|
||||
"the prepared fence must not invalidate an already leased operation"
|
||||
);
|
||||
.expect("owned update should revoke before caller cancellation");
|
||||
caller.abort();
|
||||
|
||||
let config_file = tier_config_lock_path();
|
||||
@@ -17374,71 +17026,6 @@ mod tests {
|
||||
assert!(current.tiers.contains_key("COLD-B"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("prepared-fence revision fixture should persist");
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
|
||||
}
|
||||
|
||||
let barrier = tier_reference_proof_test_barrier();
|
||||
let scoped_barrier = barrier.clone();
|
||||
let update_manager = manager.clone();
|
||||
let update_store = store.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
TIER_MUTATION_TEST_PEERS.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Remove("COLD-A".to_string(), true),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.arrived.notified().await;
|
||||
|
||||
let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
|
||||
.await
|
||||
.expect("an unrelated prepared fence should advance the runtime revision");
|
||||
barrier.release.add_permits(1);
|
||||
|
||||
let err = update
|
||||
.await
|
||||
.expect("tier update task should join")
|
||||
.expect_err("a reference proof cannot authorize publication across a fence revision change");
|
||||
let TierConfigUpdateError::Publish(err) = err else {
|
||||
panic!("the stale prepared-fence allowance should fail publication: {err:?}");
|
||||
};
|
||||
assert!(err.message.contains("changed before replacement"), "{err}");
|
||||
assert!(manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(
|
||||
load_tier_config_for_update(store)
|
||||
.await
|
||||
.expect("rejected tier config should remain readable")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A")
|
||||
);
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&manager, unrelated.mutation_id)
|
||||
.await
|
||||
.expect("unrelated test fence should clear");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() {
|
||||
|
||||
@@ -111,11 +111,10 @@ use crate::disk::{
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
use crate::io_support::bitrot::{
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
|
||||
create_deferred_bitrot_reader_with_stripe_handle,
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
|
||||
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
|
||||
object_mmap_read_max_length,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
|
||||
use crate::set_disk::runtime_sources;
|
||||
use crate::set_disk::shard_source::ShardReadCost;
|
||||
use crate::storage_api_contracts::object::ObjectOperations;
|
||||
@@ -3663,22 +3662,22 @@ async fn rollback_failed_rename(
|
||||
let object = object.to_string();
|
||||
let disk_namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let _namespace_commit_guard = disk_namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = disk_namespace_commit_guard;
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _task_guard = SetDisks::rename_fanout_task_guard(&object);
|
||||
SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
|
||||
#[cfg(test)]
|
||||
rollback_fault_injection::before_undo(&object, disk_index)?;
|
||||
disk.undo_write_with_namespace_owner(
|
||||
disk.delete_version(
|
||||
&bucket,
|
||||
&object,
|
||||
fi,
|
||||
false,
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
old_data_dir: rollback_dir,
|
||||
..Default::default()
|
||||
},
|
||||
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -4238,7 +4237,7 @@ impl SetDisks {
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = namespace_commit_guard;
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
let result = std::panic::AssertUnwindSafe(async {
|
||||
#[allow(clippy::let_unit_value)]
|
||||
@@ -4273,13 +4272,7 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
let rejected_before_publication = observed.rejected_before_publication();
|
||||
@@ -4608,7 +4601,7 @@ impl SetDisks {
|
||||
// Keep the storage-owned movement permit attached to the actual
|
||||
// fan-out owner, even if the caller future is cancelled.
|
||||
let _fanout_publication_scope = fanout_publication_scope;
|
||||
let _namespace_commit_guard = fanout_namespace_commit_guard.clone();
|
||||
let _namespace_commit_guard = fanout_namespace_commit_guard;
|
||||
let successful_rename_completion_rank =
|
||||
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
|
||||
let futures = fanout_disks
|
||||
@@ -4623,7 +4616,6 @@ impl SetDisks {
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let publication_scope = scanner_publication_commit_scope.clone();
|
||||
let namespace_commit_guard = fanout_namespace_commit_guard.clone();
|
||||
|
||||
async move {
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4676,13 +4668,7 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await;
|
||||
let rejected_before_publication = observed.rejected_before_publication();
|
||||
@@ -10873,358 +10859,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) {
|
||||
use crate::disk::os;
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "namespace-physical-tail";
|
||||
let object = "inline-overwrite";
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
prepare_rename_source_dirs(&dirs, &disks, "source").await;
|
||||
let mut old = metadata_test_fileinfo(object);
|
||||
old.mod_time = Some(OffsetDateTime::now_utc());
|
||||
old.size = 15;
|
||||
old.parts.clear();
|
||||
old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None);
|
||||
old.data = Some(Bytes::from_static(b"old-inline-body"));
|
||||
old.set_inline_data();
|
||||
old.metadata.insert("etag".to_string(), "old-etag".to_string());
|
||||
let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag");
|
||||
let mut hooks = Vec::new();
|
||||
let mut entered = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut publication_paths = Vec::new();
|
||||
for (disk, info) in disks.iter().flatten().zip(&mut infos) {
|
||||
disk.write_metadata(bucket, bucket, object, old.clone())
|
||||
.await
|
||||
.expect("the old inline version must be readable before overwrite");
|
||||
info.size = 11;
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical publication fixture requires local disks");
|
||||
};
|
||||
// Linux IO paths use a mount FD, which is also the namespace lock key.
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, object)
|
||||
.expect("the publication path must resolve through the disk's mount lease");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
// Sender drop also releases the syscall when an earlier assertion fails.
|
||||
let _ = release_rx.recv();
|
||||
},
|
||||
));
|
||||
entered.push(entered_rx);
|
||||
releases.push(release_tx);
|
||||
publication_paths.push(destination);
|
||||
}
|
||||
let namespace_owner = ctx.begin_namespace_commit();
|
||||
let namespace_probe = Arc::downgrade(&namespace_owner);
|
||||
let receipt = RenameRollbackReceipt::default();
|
||||
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
|
||||
&disks,
|
||||
(RUSTFS_META_TMP_BUCKET, "source"),
|
||||
infos,
|
||||
(bucket, object),
|
||||
allow_early_ack,
|
||||
RenameDataFenceOptions::new(3, None)
|
||||
.with_rollback_receipt(receipt.clone())
|
||||
.with_namespace_commit_guard(Some(namespace_owner)),
|
||||
));
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
signals = join_all(entered) => {
|
||||
assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter");
|
||||
}
|
||||
_ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all four prepared metadata renames must reach their blocking syscall");
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 1);
|
||||
|
||||
// Every wrapper timer exists before advancing; the physical closures stay blocked.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), rename)
|
||||
.await
|
||||
.expect("ordinary disk timeout must not wait for the physical rename");
|
||||
assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum");
|
||||
let report = receipt.0.get().expect("failed fanout must finish rollback accounting");
|
||||
assert_eq!(report.disks.len(), DISKS);
|
||||
assert!(
|
||||
report
|
||||
.disks
|
||||
.iter()
|
||||
.all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout)))
|
||||
);
|
||||
let pending_before_release = ctx.namespace_commits_pending();
|
||||
let owner_alive_before_release = namespace_probe.upgrade().is_some();
|
||||
let old_snapshot_generation = ctx.namespace_commit_generation();
|
||||
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
|
||||
let root = disk.path();
|
||||
assert!(
|
||||
os::acquire_rename_data_mutation_lease(&root, bucket, destination)
|
||||
.now_or_never()
|
||||
.is_none(),
|
||||
"the physical publication must still own object serialization after the async timeout"
|
||||
);
|
||||
assert!(
|
||||
root.join(RUSTFS_META_TMP_BUCKET)
|
||||
.join("source")
|
||||
.join(STORAGE_FORMAT_FILE)
|
||||
.exists()
|
||||
);
|
||||
let stored = disk
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a scanner can still read the complete old metadata while publication is paused");
|
||||
assert_eq!(stored.size, 15);
|
||||
assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice()));
|
||||
}
|
||||
assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation);
|
||||
|
||||
// Drain real syscalls before checking the regression, including on the RED run.
|
||||
drop(releases);
|
||||
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
|
||||
let root = disk.path();
|
||||
let lease = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&root, bucket, destination),
|
||||
)
|
||||
.await
|
||||
.expect("released physical publishers must drain");
|
||||
drop(lease);
|
||||
}
|
||||
for dir in &dirs {
|
||||
let reopened = reopen_local_disk(dir).await;
|
||||
let stored = reopened
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the detached prepared rename must actually publish after timeout");
|
||||
assert_eq!(stored.size, 11);
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
// The lease releases its locks before dropping the namespace owner, and the
|
||||
// owner's `Drop` runs after its `Weak` probe stops upgrading, so wait for the
|
||||
// pending counter itself instead of asserting it right after the drain.
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("released physical publishers must release namespace ownership");
|
||||
let generation_after_publication = ctx.namespace_commit_generation();
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(namespace_probe.upgrade().is_none());
|
||||
assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence");
|
||||
assert!(
|
||||
pending_before_release && owner_alive_before_release,
|
||||
"physical publication outlived namespace accounting: early_ack={allow_early_ack}, \
|
||||
pending={pending_before_release}, owner_alive={owner_alive_before_release}, \
|
||||
old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}"
|
||||
);
|
||||
assert!(
|
||||
generation_after_publication > old_snapshot_generation,
|
||||
"physical completion must invalidate the scanner's old metadata snapshot"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_namespace_owner_survives_physical_publication_timeout(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_namespace_owner_survives_physical_publication_timeout(true).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() {
|
||||
use crate::disk::os;
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
let bucket = "physical-ack-tail";
|
||||
let object = "ack-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
|
||||
prepare_rename_source_dirs(&dirs, &disks, "source").await;
|
||||
let mut infos = rename_commit_fileinfos(object, 4, "new-etag");
|
||||
for info in &mut infos {
|
||||
info.size = 11;
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
}
|
||||
let disk = disks[3].as_ref().expect("tail disk");
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let owner_probe = Arc::downgrade(&owner);
|
||||
let receipt = RenameRollbackReceipt::default();
|
||||
let caller_receipt = receipt.clone();
|
||||
let caller_disks = disks.clone();
|
||||
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let commit = SetDisks::rename_data_owned_with_fence(
|
||||
&caller_disks,
|
||||
(RUSTFS_META_TMP_BUCKET, "source"),
|
||||
infos,
|
||||
(bucket, object),
|
||||
true,
|
||||
RenameDataFenceOptions::new(3, None)
|
||||
.with_namespace_commit_guard(Some(owner))
|
||||
.with_rollback_receipt(caller_receipt),
|
||||
)
|
||||
.await
|
||||
.expect("three real disk publications must produce a successful ACK");
|
||||
assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK");
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let mut commit = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
entered_rx.await.expect("physical tail entry");
|
||||
ack_rx
|
||||
.await
|
||||
.expect("ACK must arrive while the fourth disk is physically paused")
|
||||
})
|
||||
.await
|
||||
.expect("successful quorum ACK must not wait for its physical tail");
|
||||
assert_eq!(commit.online_disks.iter().flatten().count(), 3);
|
||||
assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK");
|
||||
let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle");
|
||||
drop(commit);
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled());
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain)
|
||||
.await
|
||||
.expect("ordinary tail timeout stays bounded after ACK")
|
||||
.expect("tail owner must not panic")
|
||||
.expect("successful ACK keeps its convergence result");
|
||||
assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
|
||||
assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback");
|
||||
let pending = ctx.namespace_commits_pending();
|
||||
let alive = owner_probe.upgrade().is_some();
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
for disk in disks.iter().flatten().take(3) {
|
||||
let stored = disk
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("all ACK voters keep the new object after caller cancellation");
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
drop(release_tx);
|
||||
let lease = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination),
|
||||
)
|
||||
.await
|
||||
.expect("late physical tail drains");
|
||||
drop(lease);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late physical tail must release namespace ownership");
|
||||
for dir in &dirs {
|
||||
let stored = reopen_local_disk(dir)
|
||||
.await
|
||||
.read_version(
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("successful ACK remains committed on every disk after late publication");
|
||||
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
|
||||
}
|
||||
assert!(
|
||||
pending && alive,
|
||||
"physical ACK tail must retain namespace ownership after the coordinator exits"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(owner_probe.upgrade().is_none());
|
||||
assert!(ctx.namespace_commit_generation() > generation);
|
||||
assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
|
||||
|
||||
@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
|
||||
@@ -4498,12 +4498,6 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Read the persisted bucket identity through this set's metadata owner.
|
||||
/// Missing or non-authoritative legacy identities remain errors.
|
||||
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
|
||||
/// Admit one short scanner cache publication under this set's instance
|
||||
/// movement fence. The caller must hold the returned guard through its
|
||||
/// final conditional cache write; no scan-round work belongs under it.
|
||||
|
||||
@@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
}
|
||||
|
||||
async fn transition_cleanup_store(ctx: &Arc<crate::runtime::instance::InstanceContext>) -> Option<Arc<ECStore>> {
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_transition_cleanup_store().await;
|
||||
|
||||
transition_object_store(ctx).await
|
||||
@@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Default)]
|
||||
struct TransitionCleanupStoreBarrierState {
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
/// One-shot test barrier placed before transition cleanup resolves its ECStore.
|
||||
pub(crate) struct TransitionCleanupStoreBarrier {
|
||||
state: Arc<TransitionCleanupStoreBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCleanupStoreBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the process-local barrier for the next cleanup-store resolution.
|
||||
pub(crate) fn install() -> Self {
|
||||
@@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for TransitionCleanupStoreBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn pause_transition_cleanup_store() {
|
||||
let barrier = TRANSITION_CLEANUP_STORE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
@@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TransitionUploadedCommitBarrier {
|
||||
state: Arc<TransitionUploadedCommitBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionUploadedCommitBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl TransitionUploadedCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedCommitBarrierState {
|
||||
@@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionUploadedCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) {
|
||||
let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
upload_cleanup.update_cleanup_transaction(&transaction);
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pause_after_transition_uploaded_persisted(bucket, object).await;
|
||||
|
||||
let commit_opts = opts.as_commit_opts();
|
||||
@@ -12672,65 +12672,6 @@ mod metadata_mutation_generation_tests {
|
||||
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
||||
async fn segment_observation_equal_size_mutations_retire_metadata_generation() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "segment-observation-bucket";
|
||||
let object = "hot/object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create segment fixture bucket");
|
||||
}
|
||||
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await;
|
||||
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
|
||||
let mut replacement = PutObjReader::from_vec(b"after!".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("commit same-length replacement with normal owner locking");
|
||||
assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation");
|
||||
assert_retired(&set_disks, &old_key).await;
|
||||
drop(probe);
|
||||
let after = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read replacement metadata");
|
||||
assert_eq!(before.size, after.size);
|
||||
assert_ne!(before.etag, after.etag, "equal size is not equal content");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read replacement body through the owner");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("drain replacement body");
|
||||
assert_eq!(body, b"after!");
|
||||
drop(reader);
|
||||
|
||||
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await;
|
||||
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit metadata-only mutation with normal owner locking");
|
||||
assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences");
|
||||
assert_retired(&set_disks, &old_key).await;
|
||||
let after = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read committed metadata-only mutation");
|
||||
assert_eq!(before.size, after.size);
|
||||
assert_eq!(before.etag, after.etag);
|
||||
assert!(!before.user_defined.contains_key("x-amz-meta-segment"));
|
||||
assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
||||
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
|
||||
|
||||
@@ -630,27 +630,14 @@ impl ECStore {
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
let (meta, pool_meta_replica_state) = {
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
load.await?
|
||||
load_pool_meta_for_startup(self.pools.clone(), &mut write_state).await?
|
||||
};
|
||||
let update = meta.validate(self.pools.clone())?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": pool_meta_replica_state.needs_repair,
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
@@ -662,17 +649,15 @@ impl ECStore {
|
||||
// distributed startup can race on the same lock and replay the prior init bug.
|
||||
{
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
let persist = persist_pool_meta_for_startup_if_safe(
|
||||
installed_pool_meta = persist_pool_meta_for_startup_if_safe(
|
||||
&installed_pool_meta,
|
||||
self.pools.clone(),
|
||||
pool_meta_replica_state,
|
||||
&mut write_state,
|
||||
update,
|
||||
should_persist_pool_meta,
|
||||
);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
|
||||
installed_pool_meta = persist.await?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
{
|
||||
@@ -840,11 +825,6 @@ mod tests {
|
||||
manual_transition_scope_record_object_name, manual_transition_task_object_name,
|
||||
manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
|
||||
},
|
||||
recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
|
||||
IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
|
||||
observe_recovery_source, save_recovery_control_if_absent,
|
||||
},
|
||||
tier_delete_journal::{
|
||||
DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
|
||||
TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
|
||||
@@ -864,13 +844,12 @@ mod tests {
|
||||
},
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
|
||||
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
load_transition_transaction_record, recover_transition_transaction_records,
|
||||
recover_transition_transaction_records_at, save_transition_transaction_record,
|
||||
save_transition_transaction_record_if_current, transition_recovery_control_id,
|
||||
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity,
|
||||
TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, recover_transition_transaction_records_at,
|
||||
save_transition_transaction_record, save_transition_transaction_record_if_current,
|
||||
transition_transaction_record_object_name,
|
||||
},
|
||||
validate_durable_ilm_record,
|
||||
@@ -16775,141 +16754,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "LEGACY-RECOVERY";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("legacy recovery tier lease should resolve")
|
||||
.backend_identity();
|
||||
let fixtures = [
|
||||
serde_json::json!({
|
||||
"version": 1,
|
||||
"obj_name": "legacy/remote-v1",
|
||||
"version_id": "opaque-v1",
|
||||
"tier_name": tier_name,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": 2,
|
||||
"obj_name": "legacy/remote-v2",
|
||||
"version_id": "opaque-v2",
|
||||
"tier_name": tier_name,
|
||||
"backend_identity": backend_identity,
|
||||
}),
|
||||
];
|
||||
let mut journal_paths = Vec::new();
|
||||
for fixture in &fixtures {
|
||||
let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
|
||||
let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
|
||||
.expect("legacy journal fixture should decode");
|
||||
let path = tier_delete_journal_object_name(&entry);
|
||||
com::save_config(store.clone(), &path, data)
|
||||
.await
|
||||
.expect("legacy journal fixture should persist");
|
||||
journal_paths.push(path);
|
||||
}
|
||||
let corrupt_path = format!(
|
||||
"{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
|
||||
rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
|
||||
);
|
||||
com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt legacy journal fixture should persist");
|
||||
|
||||
let (first, concurrent) = tokio::join!(
|
||||
recover_tier_delete_journal_entries(store.clone(), 100, None),
|
||||
recover_tier_delete_journal_entries(store.clone(), 100, None),
|
||||
);
|
||||
for stats in [first, concurrent] {
|
||||
let stats = stats.expect("concurrent legacy journal recovery scan should finish");
|
||||
assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
|
||||
}
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
|
||||
assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
|
||||
assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
|
||||
assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
|
||||
|
||||
let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
|
||||
.await
|
||||
.expect("legacy recovery controls should be listable")
|
||||
.records;
|
||||
first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
|
||||
assert_eq!(first_controls.len(), 3);
|
||||
assert_eq!(
|
||||
first_controls
|
||||
.iter()
|
||||
.filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
first_controls
|
||||
.iter()
|
||||
.filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
for view in &first_controls {
|
||||
assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
|
||||
assert_eq!(view.revision, 1);
|
||||
assert_eq!(view.attempt_count, 0);
|
||||
let encoded = serde_json::to_string(view).expect("recovery control view should encode");
|
||||
for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
|
||||
assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
|
||||
}
|
||||
let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
|
||||
.await
|
||||
.expect("legacy recovery control should load");
|
||||
match view.source_schema.as_str() {
|
||||
"rustfs-tier-delete-journal-v1" => {
|
||||
assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
|
||||
assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
|
||||
}
|
||||
"rustfs-tier-delete-journal-v2" => {
|
||||
assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
|
||||
assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
|
||||
}
|
||||
"rustfs-tier-delete-journal-unknown" => {
|
||||
assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
|
||||
assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
|
||||
}
|
||||
schema => panic!("unexpected legacy recovery source schema: {schema}"),
|
||||
}
|
||||
}
|
||||
|
||||
com::save_config(
|
||||
store.clone(),
|
||||
&journal_paths[0],
|
||||
serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("equivalent legacy journal rewrite should persist");
|
||||
let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("repeated legacy journal recovery scan should finish");
|
||||
assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
|
||||
let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
|
||||
.await
|
||||
.expect("repeated legacy recovery controls should remain listable")
|
||||
.records;
|
||||
second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
|
||||
assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
|
||||
assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
assert!(
|
||||
backend.op_log().await.is_empty(),
|
||||
"repeated recovery must not invoke any backend operation"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -19395,105 +19239,6 @@ mod tests {
|
||||
assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
"transition-transaction-expired-attempt-budget",
|
||||
&[4],
|
||||
))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Versioned,
|
||||
},
|
||||
tier_name: "UNUSEDABANDONEDTIER".to_string(),
|
||||
backend_fingerprint: [7; 32],
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let record_name =
|
||||
transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
|
||||
let source = observe_recovery_source(
|
||||
store.clone(),
|
||||
&record_name,
|
||||
crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
|
||||
)
|
||||
.await
|
||||
.expect("transaction source generation should be observable");
|
||||
let mut control = IlmRecoveryControl::new(
|
||||
IlmRecoveryControlIdentity {
|
||||
protocol: IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: record_name,
|
||||
stable_operation_identity: transaction.transaction_id.to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
},
|
||||
source.generation,
|
||||
IlmRecoveryClassification::Retrying,
|
||||
2_000_000_000,
|
||||
IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("recovery control should build");
|
||||
let mut now = 3_000_000_000;
|
||||
for _ in 1..MAX_RECOVERY_ATTEMPTS {
|
||||
now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
|
||||
control
|
||||
.claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
|
||||
.expect("abandoned attempt should claim");
|
||||
control
|
||||
.record_expired_attempt(now + 1)
|
||||
.expect("expired attempt should consume retry budget");
|
||||
now += 2;
|
||||
}
|
||||
now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
|
||||
control
|
||||
.claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
|
||||
.expect("final abandoned attempt should claim");
|
||||
save_recovery_control_if_absent(store.clone(), &control)
|
||||
.await
|
||||
.expect("claimed recovery control should persist");
|
||||
|
||||
let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
|
||||
.await
|
||||
.expect("recovery should account for the expired attempt");
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
|
||||
|
||||
let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
|
||||
let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
|
||||
.await
|
||||
.expect("expired recovery control should remain inspectable");
|
||||
assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
|
||||
assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
|
||||
assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
|
||||
assert!(persisted.control.owner.is_none());
|
||||
assert_eq!(
|
||||
transition_transaction_record_count(store).await,
|
||||
1,
|
||||
"budget exhaustion must retain the source record"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -19606,7 +19351,6 @@ mod tests {
|
||||
),
|
||||
];
|
||||
let mut expected_removes = Vec::new();
|
||||
let mut recovery_control_ids = Vec::new();
|
||||
for (case, put_version, remote_version, source_mode) in cases {
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
@@ -19644,8 +19388,6 @@ mod tests {
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
recovery_control_ids
|
||||
.push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
|
||||
expected_removes.push((transaction.remote_object, put_version));
|
||||
}
|
||||
|
||||
@@ -19661,14 +19403,6 @@ mod tests {
|
||||
assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
for control_id in recovery_control_ids {
|
||||
let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
|
||||
.await
|
||||
.expect("completed recovery control should remain inspectable");
|
||||
assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
|
||||
assert_eq!(control.control.attempt_count, 1);
|
||||
assert!(control.control.owner.is_none());
|
||||
}
|
||||
|
||||
let replay = recover_transition_transaction_records(store, 100, None)
|
||||
.await
|
||||
@@ -19681,94 +19415,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXTERMINALCRASH";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Versioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(
|
||||
transaction.fence(),
|
||||
TransitionTransactionState::Uploaded,
|
||||
Some(TransitionRemoteVersion::versioned(remote_version.clone())),
|
||||
)
|
||||
.expect("transaction should enter uploaded state");
|
||||
backend.set_put_remote_version(Some(remote_version)).await;
|
||||
let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
|
||||
|
||||
let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
|
||||
let recovery_store = store.clone();
|
||||
let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
|
||||
barrier.wait_until_paused().await;
|
||||
let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
|
||||
.await
|
||||
.expect("terminal control should persist before source cleanup");
|
||||
assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
|
||||
recovery.abort();
|
||||
assert!(
|
||||
recovery
|
||||
.await
|
||||
.expect_err("recovery should be cancelled at the crash boundary")
|
||||
.is_cancelled()
|
||||
);
|
||||
drop(barrier);
|
||||
|
||||
let replay = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("terminal control should resume source cleanup without another remote delete");
|
||||
assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store).await, 0);
|
||||
assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -19826,8 +19472,6 @@ mod tests {
|
||||
save_transition_transaction_record(store.clone(), &uploaded)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
let recovery_control_id =
|
||||
transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
|
||||
|
||||
let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
|
||||
let recovery_store = store.clone();
|
||||
@@ -19849,17 +19493,11 @@ mod tests {
|
||||
.expect("recovery should treat the lost CAS as a retained transaction");
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
|
||||
assert_eq!(
|
||||
load_transition_transaction_record(store.clone(), uploaded.transaction_id)
|
||||
load_transition_transaction_record(store, uploaded.transaction_id)
|
||||
.await
|
||||
.expect("newer transaction revision must remain"),
|
||||
active
|
||||
);
|
||||
let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
|
||||
.await
|
||||
.expect("lost source CAS should retain a retryable recovery control");
|
||||
assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(control.control.consecutive_failure_count, 1);
|
||||
assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
|
||||
assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
@@ -20161,15 +19799,27 @@ mod tests {
|
||||
not_after_unix_nanos: 1_780_000_000_000_000_000,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
let uploaded_fence = transaction
|
||||
.advance(
|
||||
transaction.fence(),
|
||||
TransitionTransactionState::Uploaded,
|
||||
Some(TransitionRemoteVersion::versioned(remote_version.clone())),
|
||||
Some(TransitionRemoteVersion::versioned(remote_version)),
|
||||
)
|
||||
.expect("transaction should enter uploaded state");
|
||||
transaction
|
||||
.mark_cleanup_pending(
|
||||
uploaded_fence,
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
write_id: transaction.write_id,
|
||||
remote_object: transaction.remote_object.clone(),
|
||||
remote_version: transaction.remote_version.clone(),
|
||||
backend_fingerprint: transaction.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
|
||||
},
|
||||
)
|
||||
.expect("transaction should enter cleanup pending state");
|
||||
let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
|
||||
backend.set_put_remote_version(Some(remote_version)).await;
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
@@ -20181,8 +19831,6 @@ mod tests {
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
let recovery_control_id =
|
||||
transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
|
||||
|
||||
backend.set_remove_failure(true);
|
||||
let stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
@@ -20198,42 +19846,6 @@ mod tests {
|
||||
assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
|
||||
.await
|
||||
.expect("failed recovery control should persist");
|
||||
assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(control.control.attempt_count, 1);
|
||||
assert_eq!(control.control.consecutive_failure_count, 1);
|
||||
assert!(
|
||||
control
|
||||
.control
|
||||
.next_attempt_at_unix_nanos
|
||||
.is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
|
||||
);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let replay = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery before the persisted deadline should be skipped");
|
||||
assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
|
||||
assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
|
||||
|
||||
let retry_at = control
|
||||
.control
|
||||
.next_attempt_at_unix_nanos
|
||||
.expect("retry deadline should persist");
|
||||
let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
|
||||
.await
|
||||
.expect("recovery at the persisted deadline should retry the advanced source generation");
|
||||
assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
|
||||
.await
|
||||
.expect("completed retry control should remain inspectable");
|
||||
assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
|
||||
assert_eq!(terminal.control.attempt_count, 2);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -20534,10 +20146,6 @@ mod tests {
|
||||
local_commit_started
|
||||
.advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
|
||||
.expect("transaction should enter local commit state");
|
||||
let upload_started_control_id =
|
||||
transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
|
||||
let local_commit_control_id =
|
||||
transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
|
||||
|
||||
backend.set_put_remote_version(Some(remote_version)).await;
|
||||
for transaction in [&upload_started, &local_commit_started] {
|
||||
@@ -20568,19 +20176,6 @@ mod tests {
|
||||
assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
let upload_started_control =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
|
||||
.await
|
||||
.expect("upload-started control should persist");
|
||||
assert_eq!(
|
||||
upload_started_control.control.classification,
|
||||
IlmRecoveryClassification::RetainedAmbiguous
|
||||
);
|
||||
let local_commit_control =
|
||||
load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
.await
|
||||
.expect("local-commit control should persist");
|
||||
assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -20919,7 +20514,7 @@ mod tests {
|
||||
.await;
|
||||
let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("active unknown ownership should remain fenced before provider recovery");
|
||||
.expect("unsupported provider recovery should fail closed");
|
||||
assert_eq!(
|
||||
(
|
||||
unsupported_stats.scanned,
|
||||
@@ -20928,28 +20523,14 @@ mod tests {
|
||||
unsupported_stats.failed
|
||||
),
|
||||
(1, 0, 1, 0),
|
||||
"active unknown ownership must retain the upload before the recovery deadline"
|
||||
"an unsupported provider probe must retain the unknown upload"
|
||||
);
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
|
||||
let recovery_control_id =
|
||||
transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
|
||||
assert!(matches!(
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
assert!(
|
||||
backend.contains(&transaction.remote_object).await,
|
||||
"active ownership must not delete the candidate"
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
|
||||
assert!(
|
||||
!backend
|
||||
.op_log()
|
||||
.await
|
||||
.iter()
|
||||
.any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
|
||||
"active ownership must not probe the provider"
|
||||
"unsupported recovery must not delete the candidate"
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup");
|
||||
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
let stats =
|
||||
|
||||
@@ -442,7 +442,7 @@ pub(crate) mod utils;
|
||||
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
@@ -1787,7 +1787,7 @@ mod tests {
|
||||
|
||||
// Build a minimal ECStore carrying an explicit instance context. Empty
|
||||
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
|
||||
pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
let endpoint_pools = EndpointServerPools::default();
|
||||
Arc::new(ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
|
||||
@@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation(
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
|
||||
data_movement_multipart_discovery_counts()
|
||||
.lock()
|
||||
@@ -246,7 +246,7 @@ impl ECStore {
|
||||
.insert(self.id, 0);
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
|
||||
data_movement_multipart_discovery_counts()
|
||||
.lock()
|
||||
|
||||
@@ -13,10 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::utils::has_bad_path_component;
|
||||
use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use tracing::{debug, error};
|
||||
|
||||
@@ -25,203 +22,6 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
|
||||
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
|
||||
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
|
||||
|
||||
/// An instance-bound capability for internal writes before ECStore/IAM startup.
|
||||
/// Its private context and volume checks cannot be replaced by a caller guard.
|
||||
#[derive(Clone)]
|
||||
pub struct BootstrapLocalTarget {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
impl BootstrapLocalTarget {
|
||||
pub fn new(ctx: Arc<InstanceContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub fn is_for_store(&self, store: &ECStore) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, &store.ctx)
|
||||
}
|
||||
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if scanner_token.is_some() {
|
||||
return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
|
||||
}
|
||||
validate_bootstrap_volume(source.0)?;
|
||||
validate_bootstrap_volume(destination.0)?;
|
||||
rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
validate_bootstrap_volume(volume)?;
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
// Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
|
||||
// Validate both raw rename volumes before any disk lookup or admission.
|
||||
if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Execute on this instance's active local disk through the physical owner.
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
|
||||
Some(Arc::new(
|
||||
self.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.map_err(|err| DiskError::other(err.to_string()))?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
// The optional ID is a cold lookup to cache only after final admission.
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
|
||||
if let Some(path) = cached_path {
|
||||
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
&& disk.get_disk_id().await? == Some(disk_id)
|
||||
{
|
||||
return Ok((disk, None));
|
||||
}
|
||||
}
|
||||
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
|
||||
// Disk identity may perform format I/O. No registry guard spans this await.
|
||||
for disk in disks {
|
||||
if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
|
||||
return Ok((disk, Some(disk_id)));
|
||||
}
|
||||
}
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
volume: &str,
|
||||
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
|
||||
if !matches!(disk.as_ref(), Disk::Local(_)) {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
let map = ctx.local_disk_map();
|
||||
let active = map.read().await;
|
||||
if !active
|
||||
.get(&disk.endpoint().to_string())
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, disk))
|
||||
{
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
// Preserve registry -> ID-cache lock order; no filesystem I/O under either.
|
||||
if let Some(disk_id) = disk_id {
|
||||
ctx.local_disk_id_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk_id, disk.endpoint().to_string());
|
||||
}
|
||||
// Admission linearizes under the registry read: replacement/quarantine
|
||||
// before this point rejects; later changes do not revoke physical I/O.
|
||||
Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit()))
|
||||
}
|
||||
|
||||
async fn rename_local_data_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?;
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn undo_local_write_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
if !opts.undo_write {
|
||||
return Err(DiskError::other("target undo requires undo_write"));
|
||||
}
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?;
|
||||
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
|
||||
.await;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
|
||||
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
|
||||
}
|
||||
@@ -465,522 +265,6 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
|
||||
let mut format = crate::layout::format::FormatV3::new(1, 1);
|
||||
format.erasure.this = id;
|
||||
format.erasure.sets[0][0] = id;
|
||||
let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("create format volume");
|
||||
tokio::fs::write(
|
||||
meta.join(crate::disk::FORMAT_CONFIG_FILE),
|
||||
serde_json::to_vec(&format).expect("encode format"),
|
||||
)
|
||||
.await
|
||||
.expect("write real disk identity");
|
||||
let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open real local disk");
|
||||
assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
disk
|
||||
}
|
||||
|
||||
fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 1, 0);
|
||||
fi.erasure.index = 1;
|
||||
fi.version_id = Some(version);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.size = i64::try_from(body.len()).expect("fixture length");
|
||||
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||
number: 1,
|
||||
size: body.len(),
|
||||
actual_size: fi.size,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.data = Some(bytes::Bytes::from_static(body));
|
||||
fi.set_inline_data();
|
||||
fi
|
||||
}
|
||||
|
||||
async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
|
||||
let dir = disk.path().join(volume);
|
||||
tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
|
||||
disk.write_metadata(volume, volume, object, fi.clone())
|
||||
.await
|
||||
.expect("seed real metadata");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
object,
|
||||
&fi.version_id.expect("fixture version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read fixture before mutation");
|
||||
assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
|
||||
tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
let ctx_a = Arc::new(InstanceContext::new());
|
||||
let ctx_b = Arc::new(InstanceContext::new());
|
||||
let a = tempfile::tempdir().expect("A root");
|
||||
let b = tempfile::tempdir().expect("B root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk_a = target_disk(&ctx_a, a.path(), id).await;
|
||||
let disk_b = target_disk(&ctx_b, b.path(), id).await;
|
||||
if warm {
|
||||
assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
|
||||
assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
|
||||
}
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("destination", version, b"new-A");
|
||||
for disk in [&disk_a, &disk_b] {
|
||||
seed_target(disk, "target-bucket", "staged", fi.clone()).await;
|
||||
}
|
||||
let b_before = seed_target(
|
||||
&disk_b,
|
||||
"target-bucket",
|
||||
"destination",
|
||||
target_file_info("destination", version, b"old-B"),
|
||||
)
|
||||
.await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
|
||||
store
|
||||
.rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
|
||||
.await
|
||||
.expect("rename on A");
|
||||
let read = disk_a
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read committed A");
|
||||
assert_eq!(read.data, fi.data, "warm={warm}");
|
||||
assert_eq!(
|
||||
tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
|
||||
.await
|
||||
.expect("B metadata"),
|
||||
b_before
|
||||
);
|
||||
assert!(b.path().join("target-bucket/staged/xl.meta").exists());
|
||||
assert!(ctx_a.namespace_commit_generation() > 0);
|
||||
assert_eq!(ctx_b.namespace_commit_generation(), 0);
|
||||
assert!(!ctx_a.namespace_commits_pending());
|
||||
assert!(!ctx_b.namespace_commits_pending());
|
||||
assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
for state in ["removed", "quarantined", "replaced"] {
|
||||
let replacement = new_disk(
|
||||
&disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("separate active Arc");
|
||||
let map = ctx.local_disk_map();
|
||||
let mut entries = map.write().await;
|
||||
match state {
|
||||
"removed" => {
|
||||
entries.remove(&endpoint);
|
||||
}
|
||||
"quarantined" => {
|
||||
entries.insert(endpoint.clone(), None);
|
||||
}
|
||||
_ => {
|
||||
entries.insert(endpoint.clone(), Some(replacement));
|
||||
}
|
||||
}
|
||||
drop(entries);
|
||||
assert!(
|
||||
matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)),
|
||||
"{state}"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let old_id = Uuid::new_v4();
|
||||
let old = target_disk(&ctx, root.path(), old_id).await;
|
||||
assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
|
||||
let replacement_id = Uuid::new_v4();
|
||||
let replacement = target_disk(&ctx, root.path(), replacement_id).await;
|
||||
assert!(!Arc::ptr_eq(&old, &replacement));
|
||||
assert!(matches!(
|
||||
local_disk_candidate(&ctx, &old_id.to_string()).await,
|
||||
Err(DiskError::DiskNotFound)
|
||||
));
|
||||
let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
|
||||
.await
|
||||
.expect("replacement UUID");
|
||||
assert!(Arc::ptr_eq(&candidate, &replacement));
|
||||
assert_eq!(verified, Some(replacement_id));
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let target = BootstrapLocalTarget::new(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"body");
|
||||
let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
|
||||
let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
|
||||
for invalid in [
|
||||
"victim",
|
||||
".rustfs.sys/../victim",
|
||||
".rustfs.sys/./tmp",
|
||||
".rustfs.sys/ .. /victim",
|
||||
".rustfs.sys\\..\\victim",
|
||||
".minio.sys/../victim",
|
||||
] {
|
||||
for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
|
||||
.await
|
||||
.is_err(),
|
||||
"src={src}, dst={dst}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.undo_local_write(
|
||||
&disk.endpoint().to_string(),
|
||||
invalid,
|
||||
"staged",
|
||||
fi.clone(),
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"{invalid}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
(".rustfs.sys/tmp", "staged"),
|
||||
&fi,
|
||||
(".rustfs.sys/tmp", "destination"),
|
||||
Some(Uuid::new_v4())
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("victim/staged/xl.meta"))
|
||||
.await
|
||||
.expect("user source"),
|
||||
user_before
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
|
||||
.await
|
||||
.expect("metadata source"),
|
||||
meta_before
|
||||
);
|
||||
assert!(!root.path().join("victim/destination").exists());
|
||||
assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
|
||||
for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
|
||||
seed_target(&disk, volume, "staged", fi.clone()).await;
|
||||
BootstrapLocalTarget::new(ctx.clone())
|
||||
.rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
|
||||
.await
|
||||
.expect("legitimate bootstrap metadata write");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read bootstrap result");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let sibling = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let (token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("real scanner token in A");
|
||||
let destination = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local disk")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&destination, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token),
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical rename entered"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
drop(rename);
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(!sibling.namespace_commits_pending());
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
|
||||
.await
|
||||
.is_err(),
|
||||
"real pending rename blocks another scanner publication"
|
||||
);
|
||||
assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
|
||||
let gate = ctx.data_movement_operation_gate();
|
||||
assert!(
|
||||
gate.clone().try_write_owned().is_err(),
|
||||
"physical operation still owns the scanner read guard"
|
||||
);
|
||||
drop(release_tx);
|
||||
let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
|
||||
.await
|
||||
.expect("physical tail must release scanner guard");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("namespace owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual late commit");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert!(ctx.namespace_commit_generation() >= 2);
|
||||
assert_eq!(sibling.namespace_commit_generation(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let other = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let other_store = super::super::tests::build_store_with_ctx(other);
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
|
||||
let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
|
||||
let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
|
||||
let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
|
||||
assert!(store.release_scanner_publication_lease(released).await);
|
||||
let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
|
||||
for token in [Uuid::new_v4(), foreign, released] {
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(valid)
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"expired real token"
|
||||
);
|
||||
let _ = other_store.release_scanner_publication_lease(foreign).await;
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
|
||||
.await
|
||||
.expect("source bytes"),
|
||||
before
|
||||
);
|
||||
assert!(!root.path().join("target-bucket/destination").exists());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
None,
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical entry"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded entry");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||
.await
|
||||
.expect("ordinary deadline remains enabled");
|
||||
assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
|
||||
drop(rename);
|
||||
assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late physical owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual timeout tail");
|
||||
assert_eq!(read.data, fi.data);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
|
||||
@@ -54,15 +54,6 @@ pub enum Error {
|
||||
#[error("Heal task execution failed: {message}")]
|
||||
TaskExecutionFailed { message: String },
|
||||
|
||||
/// The current page already exhausted its local retry budget. Retrying
|
||||
/// the enclosing bucket would replay pages whose results were counted.
|
||||
#[error("Heal listing failed for bucket {bucket}: {source}")]
|
||||
HealListingFailed {
|
||||
bucket: String,
|
||||
#[source]
|
||||
source: Box<Error>,
|
||||
},
|
||||
|
||||
#[error("Invalid heal type: {heal_type}")]
|
||||
InvalidHealType { heal_type: String },
|
||||
|
||||
|
||||
@@ -447,7 +447,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
None,
|
||||
@@ -464,7 +463,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
|
||||
@@ -481,7 +479,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"finished".to_string(),
|
||||
None,
|
||||
@@ -498,7 +495,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task cancelled".to_string()),
|
||||
@@ -515,7 +511,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task timed out".to_string()),
|
||||
@@ -532,7 +527,6 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some(error),
|
||||
|
||||
@@ -108,144 +108,6 @@ pub struct ErasureSetHealer {
|
||||
target_endpoints: Arc<[String]>,
|
||||
replacement_task_id: Option<String>,
|
||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
}
|
||||
|
||||
async fn acquire_page_permit(
|
||||
semaphore: Arc<Semaphore>,
|
||||
pacer: Option<&super::pacing::MainlinePacer>,
|
||||
cancel: &tokio_util::sync::CancellationToken,
|
||||
) -> Result<tokio::sync::OwnedSemaphorePermit> {
|
||||
let acquire = || async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => Err(Error::TaskCancelled),
|
||||
permit = semaphore.clone().acquire_owned() => permit.map_err(|err| Error::other(format!("Failed to acquire page concurrency permit: {err}"))),
|
||||
}
|
||||
};
|
||||
let mut paid_pause = false;
|
||||
loop {
|
||||
let permit = acquire().await?;
|
||||
if let Some(pacer) = pacer {
|
||||
// Keep the real permit on the low-pressure path. Every acquisition
|
||||
// gets a fresh decision, including a waiter that queued a second
|
||||
// time. One completed pause is a bounded minimum-progress grant.
|
||||
match pacer.admission_decision() {
|
||||
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
|
||||
drop(permit);
|
||||
paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
return Ok(permit);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mainline_pacing_tests {
|
||||
use super::*;
|
||||
use crate::heal::pacing::{MainlinePacer, TestPressure};
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_page_waiters_resample_after_capacity_and_release_permits() {
|
||||
let semaphore = Arc::new(Semaphore::new(1));
|
||||
let occupied = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("existing object owns capacity");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pacer = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250)).expect("pacer"));
|
||||
let cancel = CancellationToken::new();
|
||||
let waiting = tokio::spawn({
|
||||
let semaphore = semaphore.clone();
|
||||
let pacer = pacer.clone();
|
||||
let cancel = cancel.clone();
|
||||
async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
drop(occupied);
|
||||
provider.sampled.notified().await;
|
||||
assert_eq!(semaphore.available_permits(), 1, "running pressure wait cannot retain the page permit");
|
||||
cancel.cancel();
|
||||
assert!(matches!(waiting.await.expect("page waiter"), Err(Error::TaskCancelled)));
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
let deadline = tokio::time::timeout(
|
||||
Duration::from_millis(10),
|
||||
acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(deadline.is_err());
|
||||
assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
|
||||
let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
|
||||
.await
|
||||
.expect("unpaced admission");
|
||||
assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
|
||||
drop(permit);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_two_page_waiters_check_pressure_at_final_admission() {
|
||||
use std::task::Poll;
|
||||
for raise_pressure in [false, true] {
|
||||
let semaphore = Arc::new(Semaphore::new(1));
|
||||
let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause).expect("pacer");
|
||||
let cancel = CancellationToken::new();
|
||||
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
drop(occupied);
|
||||
let first_ready = match futures::poll!(first.as_mut()) {
|
||||
Poll::Ready(result) => Some(result.expect("first admission")),
|
||||
Poll::Pending => None,
|
||||
};
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
let first_permit = match first_ready {
|
||||
Some(permit) => permit,
|
||||
None => tokio::time::timeout(Duration::from_millis(1), first)
|
||||
.await
|
||||
.expect("low-pressure waiters must not bounce capacity forever")
|
||||
.expect("first permit"),
|
||||
};
|
||||
// The first object owns real page capacity while its commit runs.
|
||||
tokio::time::advance(Duration::from_millis(100)).await;
|
||||
if raise_pressure {
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
}
|
||||
drop(first_permit);
|
||||
let admitted = if raise_pressure {
|
||||
assert!(
|
||||
futures::poll!(second.as_mut()).is_pending(),
|
||||
"a second acquisition cannot reuse the earlier low-pressure sample"
|
||||
);
|
||||
assert_eq!(semaphore.available_permits(), 1, "pressure wait must release page capacity");
|
||||
tokio::time::advance(pause).await;
|
||||
tokio::time::timeout(Duration::from_millis(1), second)
|
||||
.await
|
||||
.expect("sustained pressure must allow one unit after its bounded pause")
|
||||
.expect("second permit")
|
||||
} else {
|
||||
tokio::time::timeout(Duration::from_millis(1), second)
|
||||
.await
|
||||
.expect("low pressure must make progress")
|
||||
.expect("second permit")
|
||||
};
|
||||
assert_eq!(semaphore.available_permits(), 0);
|
||||
drop(admitted);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn target_outcomes_complete(result: &HealResultItem, target_endpoints: &[String]) -> bool {
|
||||
@@ -357,15 +219,9 @@ impl ErasureSetHealer {
|
||||
target_endpoints: Vec::new().into(),
|
||||
replacement_task_id: None,
|
||||
replacement_target_identities: None,
|
||||
mainline_pacer: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_replacement_targets(
|
||||
mut self,
|
||||
mut target_endpoints: Vec<String>,
|
||||
@@ -1000,9 +856,6 @@ impl ErasureSetHealer {
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
if let Some(pacer) = &self.mainline_pacer {
|
||||
pacer.wait(&self.cancel_token).await?;
|
||||
}
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
// Get one page of object versions
|
||||
let (objects, next_token, is_truncated) = if use_disk_walk {
|
||||
@@ -1181,10 +1034,13 @@ impl ErasureSetHealer {
|
||||
let semaphore = semaphore.clone();
|
||||
let target_endpoints = self.target_endpoints.clone();
|
||||
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
|
||||
let mainline_pacer = self.mainline_pacer.clone();
|
||||
|
||||
page_tasks.push(async move {
|
||||
let permit = acquire_page_permit(semaphore, mainline_pacer.as_deref(), &cancel_token).await;
|
||||
let permit = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")));
|
||||
|
||||
let _permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::heal::{
|
||||
outcome::HealTaskOutcome,
|
||||
progress::{HealProgress, HealStatistics},
|
||||
resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
|
||||
storage::HealStorageAPI,
|
||||
@@ -186,7 +185,6 @@ fn record_displaced_terminal(
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
outcome: None,
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
@@ -270,7 +268,6 @@ async fn publish_completed_heal(
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealTaskReport {
|
||||
pub outcome: Option<Arc<HealTaskOutcome>>,
|
||||
pub status: HealTaskStatus,
|
||||
pub result_items: Vec<HealResultItem>,
|
||||
pub result_items_truncated: bool,
|
||||
@@ -288,7 +285,6 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
|
||||
let window = task.get_result_items_since(since).await;
|
||||
HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
outcome: Some(Arc::new(task.get_outcome().await)),
|
||||
result_items: window.items,
|
||||
// The legacy flag stays set once anything was evicted; a lagging
|
||||
// incremental cursor additionally marks this response truncated so
|
||||
@@ -302,7 +298,6 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
|
||||
|
||||
fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
|
||||
HealTaskReport {
|
||||
outcome: None,
|
||||
status,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
@@ -330,7 +325,6 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
};
|
||||
HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
outcome: completed.outcome.clone(),
|
||||
result_items,
|
||||
result_items_truncated: completed.result_items_truncated || lagged,
|
||||
progress: completed.progress.clone(),
|
||||
@@ -602,13 +596,13 @@ pub struct HealConfig {
|
||||
pub set_bulkhead_enable: bool,
|
||||
/// Whether erasure-set page parallelism is enabled.
|
||||
pub page_parallel_enable: bool,
|
||||
/// Whether foreground pressure delays best-effort starts and paces running admin work.
|
||||
/// Whether foreground read pressure can delay best-effort heal task starts.
|
||||
pub mainline_throttle_enable: bool,
|
||||
/// Foreground read utilization high watermark for start admission and admin pacing.
|
||||
/// Foreground read permit utilization percentage that delays best-effort heal starts.
|
||||
pub mainline_read_utilization_high_percent: usize,
|
||||
/// Foreground write utilization high watermark for start admission and admin pacing.
|
||||
/// Foreground write utilization percentage that delays best-effort heal starts.
|
||||
pub mainline_write_utilization_high_percent: usize,
|
||||
/// Start recheck interval; running admin pacing caps each holder's pause at one second.
|
||||
/// Delay before rechecking foreground pressure after delaying heal starts.
|
||||
pub mainline_max_sleep: Duration,
|
||||
}
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@ pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) progress: Option<HealProgress>,
|
||||
pub(super) outcome: Option<Arc<HealTaskOutcome>>,
|
||||
pub(super) retained_bytes: std::sync::OnceLock<usize>,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
@@ -106,7 +105,6 @@ impl CompletedHealStatus {
|
||||
fn measure_retained_bytes(&self) -> usize {
|
||||
let mut bytes = size_of::<Self>();
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => {}
|
||||
HealType::Bucket { bucket } => add(bucket.capacity()),
|
||||
@@ -211,7 +209,6 @@ impl CompletedHealStatus {
|
||||
heal_type: task.heal_type.clone(),
|
||||
status,
|
||||
progress: Some(task.get_progress().await),
|
||||
outcome: Some(Arc::new(task.get_outcome().await)),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
|
||||
@@ -175,23 +175,11 @@ impl HealManager {
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(&request.id)
|
||||
.cloned();
|
||||
let mainline_pacer = if request.source == HealRequestSource::Admin && config.mainline_throttle_enable {
|
||||
workload_provider.as_ref().and_then(|provider| {
|
||||
crate::heal::pacing::MainlinePacer::new(
|
||||
provider.clone(),
|
||||
config.mainline_read_utilization_high_percent,
|
||||
config.mainline_write_utilization_high_percent,
|
||||
config.mainline_max_sleep,
|
||||
)
|
||||
.map(Arc::new)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let task = Arc::new(
|
||||
HealTask::from_replacement_recovery_request(request, storage.clone(), replacement_resume_endpoint)
|
||||
.with_mainline_pacer(mainline_pacer),
|
||||
);
|
||||
let task = Arc::new(HealTask::from_replacement_recovery_request(
|
||||
request,
|
||||
storage.clone(),
|
||||
replacement_resume_endpoint,
|
||||
));
|
||||
let task_id = task.id.clone();
|
||||
active_heals_guard.insert(task_id.clone(), task.clone());
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
@@ -310,7 +298,6 @@ impl HealManager {
|
||||
if cancelled_completion {
|
||||
completed_status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
|
||||
@@ -14,9 +14,6 @@
|
||||
|
||||
use super::*;
|
||||
use crate::heal::EcstoreError;
|
||||
use crate::heal::outcome::{
|
||||
HealAbortReason, HealDeferredReason, HealExecutionOutcome, HealObjectDisposition, HealTraversalCoverage,
|
||||
};
|
||||
use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity};
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
@@ -26,8 +23,6 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
mod running_mainline;
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -108,7 +103,6 @@ struct MockStorage;
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
outcome: None,
|
||||
heal_type: HealType::Cluster,
|
||||
status: HealTaskStatus::Completed,
|
||||
progress: Some(HealProgress {
|
||||
@@ -293,59 +287,6 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
|
||||
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
|
||||
use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
|
||||
let bucket = "canonical-outcome-cancel-before-finish";
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
let retention_hook = Arc::new(CompletedRetentionHook::default());
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.to_string(), retention_hook.clone());
|
||||
hooks.insert(task_id.clone(), retention_hook.clone());
|
||||
}
|
||||
let finish_hook = Arc::new(OutcomeFinishTestHook {
|
||||
task_id: task_id.clone(),
|
||||
reached: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
*OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone());
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified())
|
||||
.await
|
||||
.expect("storage started");
|
||||
retention_hook.execute.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified())
|
||||
.await
|
||||
.expect("storage returned before outcome finalization");
|
||||
manager.cancel_task(&alias).await.expect("cancel wins publication");
|
||||
finish_hook.release.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler completes cancelled handoff");
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(
|
||||
report.outcome.as_ref().expect("frozen outcome").execution,
|
||||
HealExecutionOutcome::Aborted(HealAbortReason::Cancelled)
|
||||
);
|
||||
}
|
||||
retention_hook.finish.notify_one();
|
||||
*OUTCOME_FINISH_TEST_HOOK.lock().await = None;
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != bucket && key != &task_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
let bucket = "completed-retention-retry-cancel";
|
||||
@@ -384,10 +325,6 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(
|
||||
report.outcome.as_ref().expect("cancelled outcome retained").execution,
|
||||
crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled)
|
||||
);
|
||||
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
|
||||
}
|
||||
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
|
||||
@@ -454,7 +391,6 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
.expect("scheduler archives terminal");
|
||||
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
|
||||
let expected = task.get_progress().await;
|
||||
let expected_outcome = task.get_outcome().await;
|
||||
for token in [&task_id, &alias] {
|
||||
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
|
||||
let report = manager
|
||||
@@ -462,7 +398,6 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
.await
|
||||
.expect("terminal token remains queryable at handoff");
|
||||
assert_eq!(report.progress.as_ref(), Some(&expected));
|
||||
assert_eq!(report.outcome.as_deref(), Some(&expected_outcome));
|
||||
assert!(report.result_items.is_empty());
|
||||
match outcome {
|
||||
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
|
||||
@@ -520,15 +455,10 @@ impl HealStorageAPI for MockStorage {
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_object: &str,
|
||||
_version_id: Option<&str>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if bucket.starts_with("heal-start-retry-deadline-object-") && object == "blocked" {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
|
||||
hook.started.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
if bucket == "completed-retention-failed" {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "retention fixture failure".to_string(),
|
||||
@@ -590,35 +520,11 @@ impl HealStorageAPI for MockStorage {
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
if bucket.starts_with("heal-start-retry-deadline-") {
|
||||
if continuation_token.is_some() {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
|
||||
hook.started.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
let listing_timeout = bucket.starts_with("heal-start-retry-deadline-listing-");
|
||||
let names = if listing_timeout {
|
||||
vec!["completed"]
|
||||
} else {
|
||||
vec!["completed", "blocked"]
|
||||
};
|
||||
let objects = names
|
||||
.into_iter()
|
||||
.map(|name| crate::heal::storage::HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: None,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
})
|
||||
.collect();
|
||||
return Ok((objects, listing_timeout.then(|| "next".to_string()), listing_timeout));
|
||||
}
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -641,161 +547,6 @@ impl HealStorageAPI for MockStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) {
|
||||
for phase in ["listing", "object"] {
|
||||
let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}");
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Prefix {
|
||||
bucket: bucket.clone(),
|
||||
prefix: String::new(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: Some(if cancel {
|
||||
Duration::from_secs(60)
|
||||
} else {
|
||||
Duration::from_millis(200)
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::High,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task_id = request.id.clone();
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.clone(), Arc::clone(&hook));
|
||||
hooks.insert(task_id.clone(), Arc::clone(&hook));
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("admit deadline task");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("executor reaches blocked storage");
|
||||
let active = manager.get_task_report(&task_id).await.expect("active report");
|
||||
assert_eq!(active.progress.expect("real completed object progress").objects_healed, 1);
|
||||
if cancel {
|
||||
manager.active_heals.lock().await[&task_id].cancel_token.cancel();
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("deadline archives task");
|
||||
let report = manager.get_task_report(&task_id).await.expect("terminal report");
|
||||
assert_eq!(
|
||||
report.status,
|
||||
if cancel {
|
||||
HealTaskStatus::Cancelled
|
||||
} else {
|
||||
HealTaskStatus::Timeout
|
||||
},
|
||||
"blocked {phase}"
|
||||
);
|
||||
let progress = report.progress.expect("terminal progress retained");
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0, "interrupted object has no terminal storage result");
|
||||
assert_eq!(report.result_items.len(), 1, "completed result retained");
|
||||
let outcome = report.outcome.expect("canonical terminal outcome retained");
|
||||
assert_eq!(
|
||||
outcome.execution,
|
||||
HealExecutionOutcome::Aborted(if cancel {
|
||||
HealAbortReason::Cancelled
|
||||
} else {
|
||||
HealAbortReason::Deadline
|
||||
})
|
||||
);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.healed, 0, "legacy success supplies no authoritative repair proof");
|
||||
let completed = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "completed")
|
||||
.expect("completed object diagnostic retained");
|
||||
assert_eq!(completed.disposition, HealObjectDisposition::Unknown);
|
||||
if phase == "object" {
|
||||
let interrupted = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "blocked")
|
||||
.expect("interrupted object diagnostic retained");
|
||||
assert_eq!(
|
||||
interrupted.disposition,
|
||||
if cancel {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
}
|
||||
);
|
||||
} else {
|
||||
assert_eq!(outcome.objects.len(), 1, "an unread page cannot supply object identities");
|
||||
}
|
||||
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
|
||||
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
|
||||
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != &bucket && key != &task_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_deadline_preserves_real_executor_progress() {
|
||||
assert_heal_start_retry_control_preserves_real_executor_progress(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_cancellation_preserves_real_executor_progress() {
|
||||
assert_heal_start_retry_control_preserves_real_executor_progress(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
task_timeout: Duration::ZERO,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
|
||||
request.source = HealRequestSource::Admin;
|
||||
request.options.timeout = Some(Duration::from_secs(60));
|
||||
let task_id = request.id.clone();
|
||||
let created_at = request.created_at;
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.insert(task_id.clone(), Arc::clone(&hook));
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("admit explicit-budget task");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("real read-quorum failure prepares retry");
|
||||
let retry = manager.retrying_heals.lock().await[&task_id].request.clone();
|
||||
assert_eq!(retry.id, task_id);
|
||||
assert_eq!(retry.created_at, created_at);
|
||||
assert_eq!(retry.source, HealRequestSource::Admin);
|
||||
assert_eq!(retry.retry_attempts, 1);
|
||||
let remaining = retry.options.timeout.expect("retry retains explicit budget");
|
||||
assert!(remaining > Duration::ZERO && remaining < Duration::from_secs(60));
|
||||
assert!(matches!(
|
||||
manager.get_task_status(&task_id).await.expect("retry remains queryable"),
|
||||
HealTaskStatus::Retrying { retry_attempt: 1, .. }
|
||||
));
|
||||
manager.cancel_task(&task_id).await.expect("cancel held retry");
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id);
|
||||
}
|
||||
|
||||
struct ManagerRecoveryTestHook {
|
||||
replacement_resume_disk: DiskStore,
|
||||
listed: StdMutex<bool>,
|
||||
@@ -2225,7 +1976,6 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
task_id,
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type,
|
||||
status: HealTaskStatus::Retrying {
|
||||
@@ -2950,7 +2700,6 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
task_id.clone(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Retrying {
|
||||
@@ -2988,7 +2737,6 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -3020,7 +2768,6 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage::HealListItem;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
#[derive(Default)]
|
||||
struct PressureProbe {
|
||||
active: AtomicUsize,
|
||||
commit_open: AtomicBool,
|
||||
high_sampled: Notify,
|
||||
}
|
||||
|
||||
impl WorkloadAdmissionSnapshotProvider for PressureProbe {
|
||||
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
|
||||
assert!(
|
||||
!self.commit_open.load(Ordering::SeqCst),
|
||||
"pressure must not be sampled inside an object commit"
|
||||
);
|
||||
let active = self.active.load(Ordering::SeqCst);
|
||||
if active >= 80 {
|
||||
self.high_sampled.notify_one();
|
||||
}
|
||||
WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
|
||||
Some(active),
|
||||
None,
|
||||
Some(100),
|
||||
),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
struct RunningStorage {
|
||||
provider: Arc<PressureProbe>,
|
||||
namespace: Mutex<()>,
|
||||
io: Arc<Semaphore>,
|
||||
first_started: Notify,
|
||||
release_first: Notify,
|
||||
first_finished: Notify,
|
||||
second_finished: Notify,
|
||||
started: AtomicUsize,
|
||||
committed: AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for RunningStorage {
|
||||
async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
|
||||
Ok(Some(BucketInfo {
|
||||
name: bucket.into(),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
|
||||
Ok(HealResultItem::default())
|
||||
}
|
||||
async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
|
||||
Err(Error::other("no resume disk in bucket fixture"))
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((
|
||||
["a", "b"]
|
||||
.into_iter()
|
||||
.map(|name| HealListItem {
|
||||
name: name.into(),
|
||||
version_id: None,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
})
|
||||
.collect(),
|
||||
None,
|
||||
false,
|
||||
))
|
||||
}
|
||||
async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let permit = self.io.clone().acquire_owned().await.expect("fixture I/O permit");
|
||||
let namespace = self.namespace.lock().await;
|
||||
self.provider.commit_open.store(true, Ordering::SeqCst);
|
||||
let index = self.started.fetch_add(1, Ordering::SeqCst);
|
||||
if index == 0 {
|
||||
self.first_started.notify_one();
|
||||
self.release_first.notified().await;
|
||||
}
|
||||
self.committed.fetch_add(1, Ordering::SeqCst);
|
||||
self.provider.commit_open.store(false, Ordering::SeqCst);
|
||||
drop(namespace);
|
||||
drop(permit);
|
||||
if index == 0 {
|
||||
self.first_finished.notify_one();
|
||||
} else {
|
||||
self.second_finished.notify_one();
|
||||
}
|
||||
Ok((
|
||||
HealResultItem {
|
||||
object_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_fixture(
|
||||
provider_enabled: bool,
|
||||
pacing_enabled: bool,
|
||||
timeout: Duration,
|
||||
) -> (HealManager, Arc<RunningStorage>, Arc<PressureProbe>, Arc<HealTask>) {
|
||||
let provider = Arc::new(PressureProbe::default());
|
||||
let storage = Arc::new(RunningStorage {
|
||||
provider: provider.clone(),
|
||||
namespace: Mutex::new(()),
|
||||
io: Arc::new(Semaphore::new(1)),
|
||||
first_started: Notify::new(),
|
||||
release_first: Notify::new(),
|
||||
first_finished: Notify::new(),
|
||||
second_finished: Notify::new(),
|
||||
started: AtomicUsize::new(0),
|
||||
committed: AtomicUsize::new(0),
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
mainline_throttle_enable: pacing_enabled,
|
||||
mainline_read_utilization_high_percent: 80,
|
||||
mainline_write_utilization_high_percent: 80,
|
||||
mainline_max_sleep: Duration::from_millis(250),
|
||||
max_concurrent_heals: 1,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
provider_enabled.then(|| provider.clone() as WorkloadSnapshotProviderRef),
|
||||
);
|
||||
let mut request = bucket_request("running-mainline", HealPriority::High, HealRequestSource::Admin);
|
||||
request.options.recursive = true;
|
||||
request.options.timeout = Some(timeout);
|
||||
let task_id = request.id.clone();
|
||||
manager.submit_heal_request(request).await.expect("queue admin heal");
|
||||
process_manager_queue_once(&manager).await;
|
||||
storage.first_started.notified().await;
|
||||
let task = manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.get(&task_id)
|
||||
.cloned()
|
||||
.expect("running task");
|
||||
(manager, storage, provider, task)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_admin_resamples_after_commit_and_yields_without_io_guards() {
|
||||
let (_manager, storage, provider, _task) = start_fixture(true, true, Duration::from_secs(60)).await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
assert!(storage.provider.commit_open.load(Ordering::SeqCst));
|
||||
assert_eq!(storage.committed.load(Ordering::SeqCst), 0);
|
||||
storage.release_first.notify_one();
|
||||
storage.first_finished.notified().await;
|
||||
tokio::time::timeout(Duration::from_millis(1), provider.high_sampled.notified())
|
||||
.await
|
||||
.expect("running admin heal must re-sample rising pressure before its next object");
|
||||
assert_eq!(storage.started.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
storage.committed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"in-flight commit must finish despite pressure"
|
||||
);
|
||||
assert_eq!(storage.io.available_permits(), 1, "pacing must release I/O permits");
|
||||
assert!(storage.namespace.try_lock().is_ok(), "pacing must not hold the namespace lock");
|
||||
tokio::time::advance(Duration::from_millis(250)).await;
|
||||
storage.second_finished.notified().await;
|
||||
assert_eq!(
|
||||
storage.committed.load(Ordering::SeqCst),
|
||||
2,
|
||||
"sustained pressure must still allow bounded maintenance progress"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_missing_provider_or_disabled_pacing_preserves_progress() {
|
||||
for (provider_enabled, pacing_enabled) in [(false, true), (true, false)] {
|
||||
let (_manager, storage, provider, _task) = start_fixture(provider_enabled, pacing_enabled, Duration::from_secs(60)).await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
let before = tokio::time::Instant::now();
|
||||
storage.release_first.notify_one();
|
||||
storage.second_finished.notified().await;
|
||||
assert_eq!(storage.committed.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(tokio::time::Instant::now(), before);
|
||||
assert_eq!(storage.io.available_permits(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_cancellation_and_deadline_leave_next_object_unstarted() {
|
||||
for cancelled in [true, false] {
|
||||
let (_manager, storage, provider, task) = start_fixture(true, true, Duration::from_millis(100)).await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
storage.release_first.notify_one();
|
||||
provider.high_sampled.notified().await;
|
||||
if cancelled {
|
||||
task.cancel_token.cancel();
|
||||
} else {
|
||||
tokio::time::advance(Duration::from_millis(100)).await;
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while matches!(task.get_status().await, HealTaskStatus::Running) {
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("pacing must not mask cancellation or timeout");
|
||||
assert_eq!(storage.started.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(storage.committed.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(storage.io.available_permits(), 1);
|
||||
assert!(storage.namespace.try_lock().is_ok());
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(
|
||||
task.get_status().await,
|
||||
if cancelled {
|
||||
HealTaskStatus::Cancelled
|
||||
} else {
|
||||
HealTaskStatus::Timeout
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -16,8 +16,6 @@ pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod outcome;
|
||||
pub(crate) mod pacing;
|
||||
pub mod progress;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
|
||||
@@ -33,9 +33,6 @@ use std::collections::HashMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Explicit pending migration; never activates the production writer or GC.
|
||||
pub mod migration;
|
||||
|
||||
// Root-level control files avoid requiring a new directory before the first
|
||||
// atomic commit. They remain inside the storage owner's metadata volume.
|
||||
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
|
||||
@@ -69,23 +66,6 @@ struct Manifest {
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
|
||||
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
|
||||
bytes.extend_from_slice(MAGIC);
|
||||
bytes.push(VERSION);
|
||||
bytes.extend_from_slice(owner.as_bytes());
|
||||
bytes.extend_from_slice(&sequence.to_le_bytes());
|
||||
bytes.extend_from_slice(
|
||||
&u64::try_from(payload.len())
|
||||
.map_err(|_| SnapshotError::TooLarge)?
|
||||
.to_le_bytes(),
|
||||
);
|
||||
bytes.extend_from_slice(&Sha256::digest(payload));
|
||||
bytes.extend_from_slice(&Sha256::digest(&bytes));
|
||||
Self::decode(&bytes, payload.len())?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
|
||||
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,305 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Execution results are separate from repair responsibility. A legacy
|
||||
//! successful storage call supplies no authoritative repair receipt.
|
||||
|
||||
use std::{collections::VecDeque, time::SystemTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
const MAX_OUTCOME_ITEMS: usize = 128;
|
||||
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
|
||||
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealObjectKind {
|
||||
Object,
|
||||
Metadata,
|
||||
Decode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectIdentity {
|
||||
pub kind: HealObjectKind,
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
/// The requested version; None remains unresolved, never an absence proof.
|
||||
pub version_id: Option<String>,
|
||||
pub bucket_incarnation_id: Option<Uuid>,
|
||||
pub pool_index: Option<usize>,
|
||||
pub set_index: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealDeferredReason {
|
||||
DanglingDeleteGrace,
|
||||
TransientUsageCache,
|
||||
TransientExistenceCheck,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealFailureClass {
|
||||
Recoverable,
|
||||
RetryExhausted,
|
||||
Permanent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HealObjectDisposition {
|
||||
/// The legacy storage response does not prove the requested check or commit.
|
||||
Unknown,
|
||||
Repaired,
|
||||
VerifiedHealthy,
|
||||
AuthoritativelyAbsent,
|
||||
Deferred {
|
||||
reason: HealDeferredReason,
|
||||
retry_not_before: Option<SystemTime>,
|
||||
},
|
||||
Failed(HealFailureClass),
|
||||
Cancelled,
|
||||
DryRunObserved,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectOutcome {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
impl HealObjectOutcome {
|
||||
fn retained_bytes(&self) -> usize {
|
||||
size_of::<Self>()
|
||||
.saturating_add(self.identity.bucket.capacity())
|
||||
.saturating_add(self.identity.object.capacity())
|
||||
.saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity))
|
||||
.saturating_add(self.detail.as_ref().map_or(0, String::capacity))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealTraversalCoverage {
|
||||
#[default]
|
||||
Unknown,
|
||||
Partial,
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealAbortReason {
|
||||
Cancelled,
|
||||
Deadline,
|
||||
Untraversable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealExecutionOutcome {
|
||||
#[default]
|
||||
Pending,
|
||||
Running,
|
||||
Completed,
|
||||
CompletedWithErrors,
|
||||
Aborted(HealAbortReason),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealOutcomeCounters {
|
||||
pub processed: u64,
|
||||
pub healed: u64,
|
||||
pub unchanged: u64,
|
||||
/// Deferred, cancelled, dry-run and unverified results remain unresolved.
|
||||
pub skipped: u64,
|
||||
pub failed: u64,
|
||||
pub unknown: u64,
|
||||
pub attempt_failures: u64,
|
||||
pub overflowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealTaskOutcome {
|
||||
pub execution: HealExecutionOutcome,
|
||||
pub coverage: HealTraversalCoverage,
|
||||
pub counters: HealOutcomeCounters,
|
||||
/// A bounded diagnostic window, not a complete responsibility ledger.
|
||||
pub objects: VecDeque<HealObjectOutcome>,
|
||||
pub objects_truncated: bool,
|
||||
retained_object_bytes: usize,
|
||||
untraversable: bool,
|
||||
}
|
||||
|
||||
impl HealTaskOutcome {
|
||||
pub(crate) fn start(&mut self) {
|
||||
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
self.execution = HealExecutionOutcome::Running;
|
||||
}
|
||||
self.coverage = HealTraversalCoverage::Partial;
|
||||
}
|
||||
|
||||
pub(crate) fn attempt_failed(&mut self) {
|
||||
self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures);
|
||||
}
|
||||
|
||||
pub(crate) fn mark_untraversable(&mut self) {
|
||||
self.untraversable = true;
|
||||
self.coverage = HealTraversalCoverage::Partial;
|
||||
}
|
||||
|
||||
pub(crate) fn finish(&mut self, abort: Option<HealAbortReason>) {
|
||||
if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
return;
|
||||
}
|
||||
let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable));
|
||||
self.execution = match abort {
|
||||
Some(reason) => HealExecutionOutcome::Aborted(reason),
|
||||
None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors,
|
||||
None => HealExecutionOutcome::Completed,
|
||||
};
|
||||
self.coverage = if abort.is_none() && !self.counters.overflowed {
|
||||
HealTraversalCoverage::Complete
|
||||
} else {
|
||||
HealTraversalCoverage::Partial
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) fn record(&mut self, mut item: HealObjectOutcome) {
|
||||
use super::progress::increment_counter;
|
||||
let counters = &mut self.counters;
|
||||
counters.overflowed |= !increment_counter(&mut counters.processed);
|
||||
let counter = match item.disposition {
|
||||
HealObjectDisposition::Repaired => &mut counters.healed,
|
||||
HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::Failed(_) => &mut counters.failed,
|
||||
HealObjectDisposition::Unknown => {
|
||||
counters.overflowed |= !increment_counter(&mut counters.unknown);
|
||||
&mut counters.skipped
|
||||
}
|
||||
_ => &mut counters.skipped,
|
||||
};
|
||||
counters.overflowed |= !increment_counter(counter);
|
||||
if let Some(detail) = &mut item.detail {
|
||||
let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES);
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
self.objects_truncated |= end < detail.len();
|
||||
detail.truncate(end);
|
||||
detail.shrink_to_fit();
|
||||
}
|
||||
let bytes = item.retained_bytes();
|
||||
if bytes > MAX_OUTCOME_BYTES {
|
||||
self.objects_truncated = true;
|
||||
return;
|
||||
}
|
||||
while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES {
|
||||
let Some(oldest) = self.objects.pop_front() else { break };
|
||||
self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes());
|
||||
self.objects_truncated = true;
|
||||
}
|
||||
self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes);
|
||||
self.objects.push_back(item);
|
||||
}
|
||||
|
||||
pub(crate) fn retained_bytes(&self) -> usize {
|
||||
size_of::<Self>()
|
||||
.saturating_add(self.retained_object_bytes)
|
||||
.saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod canonical_outcome_tests {
|
||||
use super::*;
|
||||
|
||||
fn item(disposition: HealObjectDisposition) -> HealObjectOutcome {
|
||||
HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_categories_have_one_terminal_count() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for disposition in [
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::AuthoritativelyAbsent,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
},
|
||||
HealObjectDisposition::Failed(HealFailureClass::Permanent),
|
||||
HealObjectDisposition::Cancelled,
|
||||
HealObjectDisposition::DryRunObserved,
|
||||
] {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
let c = &outcome.counters;
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
|
||||
assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for _ in 0..MAX_OUTCOME_ITEMS {
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
}
|
||||
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
|
||||
assert!(!outcome.objects_truncated);
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
|
||||
assert!(outcome.objects_truncated);
|
||||
let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
|
||||
oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES);
|
||||
outcome.record(oversized);
|
||||
assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count"));
|
||||
assert_eq!(outcome.counters.failed, 1);
|
||||
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
|
||||
for _ in 0..MAX_OUTCOME_ITEMS {
|
||||
let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
|
||||
failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES));
|
||||
outcome.record(failed);
|
||||
}
|
||||
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
|
||||
assert!(outcome.objects.iter().all(|item| {
|
||||
item.detail
|
||||
.as_ref()
|
||||
.is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES)
|
||||
}));
|
||||
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.counters.processed = u64::MAX;
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
outcome.finish(None);
|
||||
assert!(outcome.counters.overflowed);
|
||||
assert_eq!(outcome.counters.processed, u64::MAX);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use rustfs_concurrency::{
|
||||
WorkloadAdmissionSnapshotProvider,
|
||||
workload::{ForegroundPressure, foreground_pressure},
|
||||
};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Default)]
|
||||
struct PacingState {
|
||||
throttled: bool,
|
||||
low_since: Option<Instant>,
|
||||
}
|
||||
|
||||
pub(crate) enum PacingDecision {
|
||||
Ready,
|
||||
Wait(Option<ForegroundPressure>),
|
||||
}
|
||||
|
||||
/// Cooperative pacing for one admin execution, not a storage admission permit.
|
||||
pub(crate) struct MainlinePacer {
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
write_high: usize,
|
||||
pause: Duration,
|
||||
state: Mutex<PacingState>,
|
||||
}
|
||||
|
||||
impl MainlinePacer {
|
||||
pub(crate) fn new(
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
write_high: usize,
|
||||
pause: Duration,
|
||||
) -> Option<Self> {
|
||||
if (read_high == 0 && write_high == 0) || pause.is_zero() {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
provider,
|
||||
read_high: read_high.min(100),
|
||||
write_high: write_high.min(100),
|
||||
pause: pause.min(Duration::from_secs(1)),
|
||||
state: Mutex::new(PacingState::default()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Fresh, nonblocking decision while the caller owns actual page capacity.
|
||||
/// A contended pacing latch is conservative, but never awaited here.
|
||||
pub(crate) fn admission_decision(&self) -> PacingDecision {
|
||||
let snapshot = self.provider.workload_admission_snapshot();
|
||||
let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
|
||||
if pressure.is_none() && self.state.try_lock().is_ok_and(|state| !state.throttled) {
|
||||
PacingDecision::Ready
|
||||
} else {
|
||||
PacingDecision::Wait(pressure)
|
||||
}
|
||||
}
|
||||
|
||||
/// Call only between storage operations, with no namespace lock or I/O
|
||||
/// permit held. The pacing-only mutex serializes starts within this task;
|
||||
/// each holder waits at most one pause so persistent pressure cannot stop
|
||||
/// all maintenance progress. Cancellation also interrupts queued waiters.
|
||||
pub(crate) async fn wait(&self, cancel: &CancellationToken) -> Result<()> {
|
||||
self.wait_after_admission(cancel, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Returns whether this unit paid a bounded pause. That grant permits one
|
||||
/// unit even if pressure persists when page capacity becomes available.
|
||||
pub(crate) async fn wait_after_admission(
|
||||
&self,
|
||||
cancel: &CancellationToken,
|
||||
observed: Option<ForegroundPressure>,
|
||||
) -> Result<bool> {
|
||||
let mut state = tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return Err(Error::TaskCancelled),
|
||||
state = self.state.lock() => state,
|
||||
};
|
||||
if observed.is_some() {
|
||||
state.throttled = true;
|
||||
state.low_since = None;
|
||||
}
|
||||
let snapshot = self.provider.workload_admission_snapshot();
|
||||
let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
|
||||
if pressure.is_some() {
|
||||
state.throttled = true;
|
||||
state.low_since = None;
|
||||
} else if state.throttled {
|
||||
let low = |high: usize| if high == 0 { 0 } else { (high * 3 / 4).max(1) };
|
||||
if foreground_pressure(&snapshot, low(self.read_high), low(self.write_high)).is_none() {
|
||||
let now = Instant::now();
|
||||
let since = state.low_since.get_or_insert(now);
|
||||
if now.duration_since(*since) >= self.pause.saturating_mul(4) {
|
||||
state.throttled = false;
|
||||
state.low_since = None;
|
||||
}
|
||||
} else {
|
||||
state.low_since = None;
|
||||
}
|
||||
}
|
||||
if !state.throttled {
|
||||
return Ok(false);
|
||||
}
|
||||
metrics::counter!(
|
||||
"rustfs_heal_mainline_throttle_total",
|
||||
"source" => "admin",
|
||||
"result" => "delayed",
|
||||
"reason" => pressure.or(observed).map_or("recovery_window", |pressure| pressure.reason())
|
||||
)
|
||||
.increment(1);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => Err(Error::TaskCancelled),
|
||||
_ = tokio::time::sleep(self.pause) => Ok(true),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TestPressure {
|
||||
pub(crate) active: std::sync::atomic::AtomicUsize,
|
||||
pub(crate) sampled: tokio::sync::Notify,
|
||||
class: rustfs_concurrency::WorkloadClass,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl TestPressure {
|
||||
pub(crate) fn new(class: rustfs_concurrency::WorkloadClass, active: usize) -> Self {
|
||||
Self {
|
||||
active: std::sync::atomic::AtomicUsize::new(active),
|
||||
sampled: tokio::sync::Notify::new(),
|
||||
class,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl WorkloadAdmissionSnapshotProvider for TestPressure {
|
||||
fn workload_admission_snapshot(&self) -> rustfs_concurrency::WorkloadAdmissionRegistrySnapshot {
|
||||
let active = self.active.load(std::sync::atomic::Ordering::SeqCst);
|
||||
self.sampled.notify_one();
|
||||
rustfs_concurrency::WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
rustfs_concurrency::WorkloadAdmissionSnapshot::new(self.class, rustfs_concurrency::AdmissionState::Open).with_counts(
|
||||
Some(active),
|
||||
None,
|
||||
Some(100),
|
||||
),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
|
||||
for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
|
||||
let provider = Arc::new(TestPressure::new(class, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = MainlinePacer::new(
|
||||
provider.clone(),
|
||||
if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
|
||||
if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
|
||||
pause,
|
||||
)
|
||||
.expect("enabled pacer");
|
||||
let cancel = CancellationToken::new();
|
||||
let now = Instant::now();
|
||||
pacer.wait(&cancel).await.expect("quiet work");
|
||||
assert_eq!(Instant::now(), now);
|
||||
// Low watermark is 30 for the configured high watermark 40.
|
||||
for utilization in [40, 29, 35, 29, 29, 29, 29] {
|
||||
provider.active.store(utilization, Ordering::SeqCst);
|
||||
let before = Instant::now();
|
||||
pacer.wait(&cancel).await.expect("bounded maintenance progress");
|
||||
assert_eq!(Instant::now() - before, pause);
|
||||
}
|
||||
let before = Instant::now();
|
||||
pacer.wait(&cancel).await.expect("stable low pressure restores unpaced work");
|
||||
assert_eq!(Instant::now(), before);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1)).is_none());
|
||||
assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO).is_none());
|
||||
let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600)).expect("pacer");
|
||||
let before = Instant::now();
|
||||
pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
|
||||
assert_eq!(Instant::now() - before, Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
let paced = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("pacer"));
|
||||
let cancel_first = CancellationToken::new();
|
||||
let first = tokio::spawn({
|
||||
let paced = paced.clone();
|
||||
let cancel = cancel_first.clone();
|
||||
async move { paced.wait(&cancel).await }
|
||||
});
|
||||
provider.sampled.notified().await;
|
||||
let cancel_second = CancellationToken::new();
|
||||
let second = tokio::spawn({
|
||||
let paced = paced.clone();
|
||||
let cancel = cancel_second.clone();
|
||||
async move { paced.wait(&cancel).await }
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
cancel_second.cancel();
|
||||
assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
|
||||
provider.active.store(0, Ordering::SeqCst);
|
||||
let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("independent task");
|
||||
let before = Instant::now();
|
||||
other_task
|
||||
.wait(&CancellationToken::new())
|
||||
.await
|
||||
.expect("another task has no inherited latch");
|
||||
assert_eq!(Instant::now(), before, "task/set pacing state must not be global");
|
||||
cancel_first.cancel();
|
||||
assert!(matches!(first.await.expect("sleeping waiter"), Err(Error::TaskCancelled)));
|
||||
tokio::time::timeout(Duration::from_secs(2), paced.wait(&CancellationToken::new()))
|
||||
.await
|
||||
.expect("pacing lock released")
|
||||
.expect("bounded work after cancellation");
|
||||
}
|
||||
}
|
||||
+17
-171
@@ -15,10 +15,6 @@
|
||||
use crate::heal::{
|
||||
DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _,
|
||||
erasure_healer::target_outcomes_complete,
|
||||
outcome::{
|
||||
HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind,
|
||||
HealObjectOutcome, HealTaskOutcome,
|
||||
},
|
||||
progress::HealProgress,
|
||||
resume::{
|
||||
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
|
||||
@@ -47,26 +43,6 @@ use uuid::Uuid;
|
||||
|
||||
use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct OutcomeFinishTestHook {
|
||||
pub(crate) task_id: String,
|
||||
pub(crate) reached: tokio::sync::Notify,
|
||||
pub(crate) release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_outcome_finish(task_id: &str) {
|
||||
let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
|
||||
if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_TASK: &str = "task";
|
||||
const LOG_SUBSYSTEM_OBJECT: &str = "object";
|
||||
@@ -418,7 +394,6 @@ pub struct HealTask {
|
||||
pub status: Arc<RwLock<HealTaskStatus>>,
|
||||
/// Progress tracking
|
||||
pub progress: Arc<RwLock<HealProgress>>,
|
||||
outcome: Arc<RwLock<HealTaskOutcome>>,
|
||||
/// Result items collected from storage heal calls, each stamped with a
|
||||
/// monotonically increasing sequence number for incremental consumption
|
||||
/// (the client passes the last seen seq back and receives only newer
|
||||
@@ -446,7 +421,6 @@ pub struct HealTask {
|
||||
pub cancel_token: tokio_util::sync::CancellationToken,
|
||||
/// Storage layer interface
|
||||
pub storage: Arc<dyn HealStorageAPI>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
@@ -486,7 +460,6 @@ impl HealTask {
|
||||
result_items_truncated: Arc::new(AtomicBool::new(false)),
|
||||
batch_failure: Arc::new(RwLock::new(None)),
|
||||
batch_failure_recorded: Arc::new(AtomicBool::new(false)),
|
||||
outcome: Arc::new(RwLock::new(HealTaskOutcome::default())),
|
||||
created_at: request.created_at,
|
||||
enqueued_at: request.enqueued_at,
|
||||
started_at: Arc::new(RwLock::new(None)),
|
||||
@@ -494,7 +467,6 @@ impl HealTask {
|
||||
task_start_instant: Arc::new(RwLock::new(None)),
|
||||
cancel_token: tokio_util::sync::CancellationToken::new(),
|
||||
storage,
|
||||
mainline_pacer: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -531,82 +503,10 @@ impl HealTask {
|
||||
task
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
|
||||
async fn pace_mainline(&self) -> Result<()> {
|
||||
if let Some(pacer) = &self.mainline_pacer {
|
||||
self.await_with_control(pacer.wait(&self.cancel_token)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn metric_type_label(&self) -> &'static str {
|
||||
self.heal_type.kind_label()
|
||||
}
|
||||
|
||||
pub async fn get_outcome(&self) -> HealTaskOutcome {
|
||||
self.outcome.read().await.clone()
|
||||
}
|
||||
|
||||
fn outcome_identity(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
pool_index: Option<usize>,
|
||||
set_index: Option<usize>,
|
||||
) -> HealObjectIdentity {
|
||||
HealObjectIdentity {
|
||||
kind: match self.heal_type {
|
||||
HealType::Metadata { .. } => HealObjectKind::Metadata,
|
||||
HealType::ECDecode { .. } => HealObjectKind::Decode,
|
||||
_ => HealObjectKind::Object,
|
||||
},
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: None,
|
||||
pool_index,
|
||||
set_index,
|
||||
}
|
||||
}
|
||||
|
||||
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
||||
let (bucket, object, version) = match &self.heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
| HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => (bucket, object, version_id.as_deref()),
|
||||
HealType::Metadata { bucket, object } => (bucket, object, None),
|
||||
_ => return None,
|
||||
};
|
||||
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
|
||||
}
|
||||
|
||||
async fn record_deferred_object(&self, reason: HealDeferredReason) {
|
||||
if let Some(identity) = self.single_object_identity() {
|
||||
let mut outcome = self.outcome.write().await;
|
||||
outcome.attempt_failed();
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition: HealObjectDisposition::Deferred {
|
||||
reason,
|
||||
retry_not_before: None,
|
||||
},
|
||||
detail: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_batch_failure(&self) -> bool {
|
||||
self.batch_failure_recorded.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -734,7 +634,6 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
|
||||
self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await;
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -834,8 +733,6 @@ impl HealTask {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_deferred_object(HealDeferredReason::TransientUsageCache).await;
|
||||
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -858,8 +755,6 @@ impl HealTask {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await;
|
||||
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -906,7 +801,6 @@ impl HealTask {
|
||||
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
|
||||
#[hotpath::measure]
|
||||
pub async fn execute(&self) -> Result<()> {
|
||||
self.outcome.write().await.start();
|
||||
// update status and timestamps atomically to avoid race conditions
|
||||
let now = SystemTime::now();
|
||||
let start_instant = Instant::now();
|
||||
@@ -947,71 +841,24 @@ impl HealTask {
|
||||
});
|
||||
self.emit_trace_task_state("started", Duration::ZERO, None);
|
||||
|
||||
let result = async {
|
||||
if self.heal_type.is_per_object() {
|
||||
self.pace_mainline().await?;
|
||||
}
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_object(bucket, object, version_id.as_deref()).await,
|
||||
HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
|
||||
HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
|
||||
let result = match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_object(bucket, object, version_id.as_deref()).await,
|
||||
HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
|
||||
HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
|
||||
|
||||
HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
#[cfg(test)]
|
||||
pause_outcome_finish(&self.id).await;
|
||||
{
|
||||
let mut outcome = self.outcome.write().await;
|
||||
if outcome.counters.processed == 0
|
||||
&& let Some(identity) = self.single_object_identity()
|
||||
{
|
||||
let disposition = match &result {
|
||||
Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved,
|
||||
Ok(()) => HealObjectDisposition::Unknown,
|
||||
Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled,
|
||||
Err(Error::TaskTimeout) => HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
},
|
||||
Err(error) => {
|
||||
outcome.attempt_failed();
|
||||
HealObjectDisposition::Failed(if error.is_recoverable_heal() {
|
||||
HealFailureClass::Recoverable
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
})
|
||||
}
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: result.as_ref().err().map(ToString::to_string),
|
||||
});
|
||||
}
|
||||
let abort = match &result {
|
||||
Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled),
|
||||
Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline),
|
||||
Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable),
|
||||
_ => None,
|
||||
};
|
||||
outcome.finish(abort);
|
||||
}
|
||||
HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
|
||||
HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
|
||||
};
|
||||
|
||||
// update completed time and status
|
||||
{
|
||||
@@ -1097,7 +944,6 @@ impl HealTask {
|
||||
|
||||
pub async fn cancel(&self) -> Result<()> {
|
||||
self.cancel_token.cancel();
|
||||
self.outcome.write().await.finish(Some(HealAbortReason::Cancelled));
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Cancelled;
|
||||
debug!(
|
||||
|
||||
@@ -34,7 +34,6 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
self.pace_mainline().await?;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
@@ -215,7 +214,6 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
failed = failed.saturating_add(1);
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
if err.is_recoverable_heal() {
|
||||
retryable = retryable.saturating_add(1);
|
||||
} else {
|
||||
@@ -262,7 +260,6 @@ impl HealTask {
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let previous_progress = self.get_progress().await;
|
||||
let mut scanned = 0u64;
|
||||
let mut healed = 0u64;
|
||||
let mut failed = 0u64;
|
||||
@@ -307,48 +304,23 @@ impl HealTask {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let mut listing_attempt = 0;
|
||||
let (objects, next_token, is_truncated) = loop {
|
||||
self.pace_mainline().await?;
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok(page) => break page,
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
};
|
||||
|
||||
let mut pending = objects;
|
||||
@@ -364,17 +336,8 @@ impl HealTask {
|
||||
let mut retry = Vec::with_capacity(pending.len());
|
||||
for item in pending {
|
||||
self.check_control_flags().await?;
|
||||
self.pace_mainline().await?;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
@@ -417,31 +380,7 @@ impl HealTask {
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
return Err(err);
|
||||
}
|
||||
_ => self.outcome.write().await.attempt_failed(),
|
||||
}
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -456,10 +395,6 @@ impl HealTask {
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -490,11 +425,6 @@ impl HealTask {
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
@@ -529,20 +459,8 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
progress.update_object_progress(scanned, healed, failed, skipped, bytes);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
@@ -557,7 +475,7 @@ impl HealTask {
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated without a continuation token is a compatibility EOF.
|
||||
// Truncated but no continuation token: end of listing.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,8 +422,7 @@ impl HealTask {
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone())
|
||||
.with_mainline_pacer(self.mainline_pacer.clone());
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone());
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
|
||||
@@ -261,8 +261,8 @@ impl HealTask {
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
read_repair: false,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
let heal_result = self
|
||||
|
||||
@@ -14,364 +14,6 @@
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
|
||||
mod canonical_outcome {
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
|
||||
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
|
||||
HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_retries_only_the_failed_listing_page() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect("second-page retry succeeds");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 2);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a", "object-b"]
|
||||
);
|
||||
assert_eq!(
|
||||
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
|
||||
[None, Some("second".to_string()), Some("second".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_listing_page_cannot_restart_the_bucket() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(4)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect_err("listing page budget exhausted");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.counters.attempt_failures, 4);
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 1);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]);
|
||||
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_failure_preserves_processed_objects_and_partial_coverage() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
fail_second_listing_page: true,
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect_err("second page cannot be traversed");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.objects[0].identity.object, "object-a");
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_preserves_cumulative_progress_across_buckets() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
list_each_bucket: true,
|
||||
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
);
|
||||
task.execute().await.expect("both buckets complete");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 4);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4));
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.bucket == "bucket-b")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"object-a".to_string(),
|
||||
(0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(),
|
||||
);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("legacy adapter retains batch failure");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1));
|
||||
assert_eq!(outcome.counters.attempt_failures, 4);
|
||||
let failed = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("failed object");
|
||||
assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted));
|
||||
let object_b_calls = {
|
||||
let calls = storage.heal_object_calls.lock().expect("calls");
|
||||
calls.iter().filter(|object| object.as_str() == "object-b").count()
|
||||
};
|
||||
assert_eq!(object_b_calls, 1);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_success_counts_one_terminal_outcome() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect("retry should recover");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "object-a")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
outcome.counters.processed,
|
||||
outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]));
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect("grace permits traversal completion");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt");
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown);
|
||||
assert!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| item.identity.bucket_incarnation_id.is_none())
|
||||
);
|
||||
assert_eq!(
|
||||
task.get_progress().await.objects_healed,
|
||||
1,
|
||||
"legacy display count remains distinct from proof"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grace_single_object_is_completed_but_deferred() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage);
|
||||
task.execute().await.expect("grace is deferred");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_run_and_transient_existence_do_not_prove_repair() {
|
||||
for transient in [false, true] {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
if transient {
|
||||
storage
|
||||
.object_exists_by_name
|
||||
.lock()
|
||||
.expect("existence fixture")
|
||||
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
||||
}
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.dry_run = !transient;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute().await.expect("observation may complete");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
if transient {
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientExistenceCheck,
|
||||
..
|
||||
}
|
||||
));
|
||||
} else {
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
|
||||
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect_err("structural bucket error");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
|
||||
["bucket-a", "bucket-b"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_and_deadline_leave_partial_coverage() {
|
||||
for cancel in [false, true] {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_heal_object: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.timeout = Some(Duration::from_secs(1));
|
||||
let task = HealTask::from_request(request, storage);
|
||||
if cancel {
|
||||
task.cancel().await.expect("cancel request");
|
||||
}
|
||||
task.execute().await.expect_err("control interruption");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
outcome.execution,
|
||||
HealExecutionOutcome::Aborted(if cancel {
|
||||
HealAbortReason::Cancelled
|
||||
} else {
|
||||
HealAbortReason::Deadline
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decode_keeps_the_requested_pool_and_set() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string()));
|
||||
request.options.pool_index = Some(2);
|
||||
request.options.set_index = Some(3);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
task.execute().await.expect("decode fixture");
|
||||
let pool_and_set = {
|
||||
let options = storage.object_heal_opts.lock().expect("storage options");
|
||||
(options[0].pool, options[0].set)
|
||||
};
|
||||
assert_eq!(pool_and_set, (Some(2), Some(3)));
|
||||
let outcome = task.get_outcome().await;
|
||||
let identity = &outcome.objects[0].identity;
|
||||
assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3)));
|
||||
assert_eq!(identity.version_id.as_deref(), Some("version-a"));
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
|
||||
}
|
||||
}
|
||||
use crate::heal::storage::{HealListItem, HealObjectInfo};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
@@ -940,10 +582,6 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
fail_second_listing_page: bool,
|
||||
recoverable_second_page_failures: Mutex<Option<usize>>,
|
||||
listing_tokens: Mutex<Vec<Option<String>>>,
|
||||
healed_objects: Mutex<Vec<String>>,
|
||||
heal_object_calls: Mutex<Vec<String>>,
|
||||
heal_object_version_ids: Mutex<Vec<Option<String>>>,
|
||||
@@ -1357,41 +995,12 @@ impl HealStorageAPI for MockStorage {
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
|
||||
self.listing_tokens
|
||||
.lock()
|
||||
.expect("listing tokens")
|
||||
.push(continuation_token.map(ToOwned::to_owned));
|
||||
if let Some(remaining) = self
|
||||
.recoverable_second_page_failures
|
||||
.lock()
|
||||
.expect("listing failures")
|
||||
.as_mut()
|
||||
{
|
||||
if continuation_token.is_none() {
|
||||
return Ok((vec![heal_item("object-a")], Some("second".to_string()), true));
|
||||
}
|
||||
if *remaining > 0 {
|
||||
*remaining -= 1;
|
||||
return Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
"page".to_string(),
|
||||
)));
|
||||
}
|
||||
return Ok((vec![heal_item("object-b")], None, false));
|
||||
}
|
||||
if self.fail_second_listing_page {
|
||||
return if continuation_token.is_none() {
|
||||
Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true))
|
||||
} else {
|
||||
Err(Error::other("listing unavailable"))
|
||||
};
|
||||
}
|
||||
if *self.truncate_without_token.lock().unwrap() {
|
||||
return Ok((vec![heal_item("object-a")], None, true));
|
||||
}
|
||||
|
||||
let mut listed = self.listed.lock().unwrap();
|
||||
if continuation_token.is_none() && (!*listed || self.list_each_bucket) {
|
||||
if continuation_token.is_none() && !*listed {
|
||||
*listed = true;
|
||||
let objects = if bucket == RUSTFS_META_BUCKET {
|
||||
vec![
|
||||
@@ -1784,8 +1393,6 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
|
||||
// A version listing can report the final page as truncated with no
|
||||
// continuation token. That is treated as end-of-listing (not an error),
|
||||
// so the returned page is healed and the pass terminates cleanly instead
|
||||
@@ -1807,16 +1414,10 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute()
|
||||
task.heal_bucket("bucket-a")
|
||||
.await
|
||||
.expect("truncated-without-token must terminate cleanly, not loop or error");
|
||||
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
|
||||
assert_eq!(
|
||||
storage.healed_objects.lock().unwrap().as_slice(),
|
||||
["object-a".to_string()],
|
||||
|
||||
+37
-192
@@ -429,27 +429,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// The cached mapping record for one user or group, looked up in the same
|
||||
/// cache partition `policy_db_set` writes it to (group / STS / regular+service
|
||||
/// user). `None` when no mapping is stored.
|
||||
pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
|
||||
let cache = self.cache.snapshot();
|
||||
if is_group {
|
||||
cache.group_policies.get(name).cloned()
|
||||
} else if user_type == UserType::Sts {
|
||||
cache.sts_policies.get(name).cloned()
|
||||
} else {
|
||||
cache.user_policies.get(name).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
/// The cached group record (members, status, own timestamp) without the
|
||||
/// mapped-policy overlay `get_group_description` applies. `None` when the
|
||||
/// group does not exist.
|
||||
pub async fn get_group_info(&self, name: &str) -> Option<GroupInfo> {
|
||||
self.cache.snapshot().groups.get(name).cloned()
|
||||
}
|
||||
|
||||
pub async fn get_policy(&self, name: &str) -> Result<Policy> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
@@ -555,17 +534,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
|
||||
self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_policy`] stamping the document with `updated_at` instead
|
||||
/// of the local clock.
|
||||
///
|
||||
/// A site-replication receiver passes the edit's source time: the next
|
||||
/// incoming revision is judged against the stored `UpdateDate`, so a
|
||||
/// local stamp would reject a newer source edit that was merely delivered
|
||||
/// later (backlog#2291). The returned stamp is the one persisted.
|
||||
pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
if name.is_empty() || policy.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -576,17 +544,18 @@ where
|
||||
.get(name)
|
||||
.map(|v| {
|
||||
let mut p = v.clone();
|
||||
p.update_at(policy.clone(), updated_at);
|
||||
p.update(policy.clone());
|
||||
p
|
||||
})
|
||||
.unwrap_or_else(|| PolicyDoc::new_at(policy, updated_at));
|
||||
.unwrap_or_else(|| PolicyDoc::new(policy));
|
||||
|
||||
self.api.save_policy_doc(name, policy_doc.clone()).await?;
|
||||
|
||||
self.cache
|
||||
.add_or_update_policy_doc(name, &policy_doc, OffsetDateTime::now_utc());
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
Ok(updated_at)
|
||||
self.cache.add_or_update_policy_doc(name, &policy_doc, now);
|
||||
|
||||
Ok(now)
|
||||
}
|
||||
|
||||
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
@@ -820,12 +789,6 @@ where
|
||||
|
||||
/// create a service account and update cache
|
||||
pub async fn add_service_account(&self, cred: Credentials) -> Result<OffsetDateTime> {
|
||||
self.add_service_account_at(cred, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::add_service_account`] stamping the identity with `updated_at`
|
||||
/// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn add_service_account_at(&self, cred: Credentials, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
if cred.access_key.is_empty() || cred.parent_user.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -837,8 +800,7 @@ where
|
||||
}
|
||||
drop(cache);
|
||||
|
||||
let mut u = UserIdentity::new(cred);
|
||||
u.update_at = Some(updated_at);
|
||||
let u = UserIdentity::new(cred);
|
||||
|
||||
self.api
|
||||
.save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
|
||||
@@ -846,22 +808,10 @@ where
|
||||
|
||||
self.update_user_with_claims(&u.credentials.access_key, u.clone())?;
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
|
||||
self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::update_service_account`] stamping the identity with
|
||||
/// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
|
||||
/// (backlog#2291).
|
||||
pub async fn update_service_account_at(
|
||||
&self,
|
||||
name: &str,
|
||||
opts: UpdateServiceAccountOpts,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let _mutation_guard = self.cache.service_account_mutation_lock().lock().await;
|
||||
let cache = self.cache.snapshot();
|
||||
let Some(ui) = cache.users.get(name).cloned() else {
|
||||
@@ -908,7 +858,13 @@ where
|
||||
}
|
||||
|
||||
if let Some(status) = opts.status {
|
||||
cr.status = account_status_flag(&status).to_owned();
|
||||
match status.as_str() {
|
||||
val if val == AccountStatus::Enabled.as_ref() => cr.status = auth::ACCOUNT_ON.to_owned(),
|
||||
val if val == AccountStatus::Disabled.as_ref() => cr.status = auth::ACCOUNT_OFF.to_owned(),
|
||||
auth::ACCOUNT_ON => cr.status = auth::ACCOUNT_ON.to_owned(),
|
||||
auth::ACCOUNT_OFF => cr.status = auth::ACCOUNT_OFF.to_owned(),
|
||||
_ => cr.status = auth::ACCOUNT_OFF.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
let mut m: HashMap<String, Value> = if token_without_expiration {
|
||||
@@ -960,8 +916,8 @@ where
|
||||
|
||||
cr.session_token = jwt_sign(&m, &cr.secret_key)?;
|
||||
|
||||
let mut u = UserIdentity::new(cr);
|
||||
u.update_at = Some(updated_at);
|
||||
let u = UserIdentity::new(cr);
|
||||
let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc);
|
||||
self.api
|
||||
.save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
|
||||
.await?;
|
||||
@@ -1193,20 +1149,6 @@ where
|
||||
Ok((policies.into_iter().collect(), update_at))
|
||||
}
|
||||
pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
|
||||
self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::policy_db_set`] stamping the mapping with `updated_at` instead
|
||||
/// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn policy_db_set_at(
|
||||
&self,
|
||||
name: &str,
|
||||
user_type: UserType,
|
||||
is_group: bool,
|
||||
policy: &str,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1226,11 +1168,10 @@ where
|
||||
self.cache.delete_user_policy(name, OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
return Ok(updated_at);
|
||||
return Ok(OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
let mut mp = MappedPolicy::new(policy);
|
||||
mp.update_at = updated_at;
|
||||
let mp = MappedPolicy::new(policy);
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
let policy_docs_cache = Arc::clone(&cache.policy_docs);
|
||||
@@ -1253,7 +1194,7 @@ where
|
||||
self.cache.add_or_update_user_policy(name, &mp, OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
pub async fn set_temp_user(&self, access_key: &str, cred: &Credentials, policy_name: Option<&str>) -> Result<OffsetDateTime> {
|
||||
@@ -1450,17 +1391,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn add_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
|
||||
self.add_user_at(access_key, args, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::add_user`] stamping the identity with `updated_at` instead of
|
||||
/// the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn add_user_at(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
if let Some(x) = users.get(access_key) {
|
||||
@@ -1478,13 +1408,12 @@ where
|
||||
_ => auth::ACCOUNT_OFF,
|
||||
}
|
||||
};
|
||||
let mut user_entry = UserIdentity::from(Credentials {
|
||||
let user_entry = UserIdentity::from(Credentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: args.secret_key.to_string(),
|
||||
status: status.to_owned(),
|
||||
..Default::default()
|
||||
});
|
||||
user_entry.update_at = Some(updated_at);
|
||||
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
@@ -1492,7 +1421,7 @@ where
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
|
||||
@@ -1670,17 +1599,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn set_user_status(&self, access_key: &str, status: AccountStatus) -> Result<OffsetDateTime> {
|
||||
self.set_user_status_at(access_key, status, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_user_status`] stamping the identity with `updated_at`
|
||||
/// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn set_user_status_at(
|
||||
&self,
|
||||
access_key: &str,
|
||||
status: AccountStatus,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if access_key.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1707,13 +1625,12 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
let mut user_entry = UserIdentity::from(Credentials {
|
||||
let user_entry = UserIdentity::from(Credentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: u.credentials.secret_key.clone(),
|
||||
status: status.to_owned(),
|
||||
..Default::default()
|
||||
});
|
||||
user_entry.update_at = Some(updated_at);
|
||||
drop(cache);
|
||||
drop(users);
|
||||
|
||||
@@ -1723,7 +1640,7 @@ where
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
fn update_user_with_claims(&self, k: &str, u: UserIdentity) -> Result<()> {
|
||||
@@ -1759,17 +1676,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn add_users_to_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
|
||||
self.add_users_to_group_at(group, members, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::add_users_to_group`] stamping the group with `updated_at`
|
||||
/// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn add_users_to_group_at(
|
||||
&self,
|
||||
group: &str,
|
||||
members: Vec<String>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if group.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1787,14 +1693,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// The group's own timestamp moves with every membership or status
|
||||
// change: site replication judges an incoming group item against it
|
||||
// (backlog#2291), so it must reflect the last change, not creation.
|
||||
// `updated_at` is the record's stamp only; the cache is published
|
||||
// with the local clock, because `LockedCache::exec` drops a write
|
||||
// whose time predates the entity's load time — a replicated edit
|
||||
// whose source time is older than this node's startup would
|
||||
// otherwise never reach the cache.
|
||||
let gi = match cache.groups.get(group) {
|
||||
Some(res) => {
|
||||
let mut gi = res.clone();
|
||||
@@ -1803,20 +1701,15 @@ where
|
||||
uniq_set.extend(members.iter().cloned());
|
||||
|
||||
gi.members = uniq_set.into_iter().collect();
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
None => {
|
||||
let mut gi = GroupInfo::new(members.clone());
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
None => GroupInfo::new(members.clone()),
|
||||
};
|
||||
drop(cache);
|
||||
|
||||
self.api.save_group_info(group, gi.clone()).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
|
||||
@@ -1826,18 +1719,13 @@ where
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
});
|
||||
now
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(now)
|
||||
}
|
||||
|
||||
pub async fn set_group_status(&self, name: &str, enable: bool) -> Result<OffsetDateTime> {
|
||||
self.set_group_status_at(name, enable, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_group_status`] stamping the group with `updated_at` instead
|
||||
/// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
|
||||
pub async fn set_group_status_at(&self, name: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1855,15 +1743,12 @@ where
|
||||
} else {
|
||||
gi.status = STATUS_DISABLED.to_owned();
|
||||
}
|
||||
gi.update_at = Some(updated_at);
|
||||
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
|
||||
// Cache publication time is the local clock, not the record stamp
|
||||
// (see `add_users_to_group_at`).
|
||||
self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
pub async fn get_group_description(&self, name: &str) -> Result<GroupDesc> {
|
||||
@@ -1933,20 +1818,6 @@ where
|
||||
name: &str,
|
||||
members: Vec<String>,
|
||||
update_cache_only: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.remove_members_from_group_at(name, members, update_cache_only, OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::remove_members_from_group`] stamping the group with
|
||||
/// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
|
||||
/// (backlog#2291).
|
||||
pub async fn remove_members_from_group_at(
|
||||
&self,
|
||||
name: &str,
|
||||
members: Vec<String>,
|
||||
update_cache_only: bool,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let cache = self.cache.snapshot();
|
||||
let mut gi = cache
|
||||
@@ -1959,14 +1830,12 @@ where
|
||||
let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
|
||||
let d: HashSet<&String> = HashSet::from_iter(members.iter());
|
||||
gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
|
||||
gi.update_at = Some(updated_at);
|
||||
|
||||
if !update_cache_only {
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
}
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
// Sample after storage completes so a concurrent reload cannot
|
||||
// make this publication older than the cache it must update.
|
||||
let now = self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(name, &gi, now);
|
||||
|
||||
@@ -1978,25 +1847,13 @@ where
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
});
|
||||
now
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
Ok(now)
|
||||
}
|
||||
|
||||
pub async fn remove_users_from_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
|
||||
self.remove_users_from_group_at(group, members, OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::remove_users_from_group`] stamping the group with `updated_at`
|
||||
/// instead of the local clock; a group delete (no members) leaves no
|
||||
/// record and returns the stamp unchanged (backlog#2291).
|
||||
pub async fn remove_users_from_group_at(
|
||||
&self,
|
||||
group: &str,
|
||||
members: Vec<String>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if group.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -2045,17 +1902,18 @@ where
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
now
|
||||
});
|
||||
|
||||
return Ok(updated_at);
|
||||
return Ok(now);
|
||||
}
|
||||
|
||||
self.remove_members_from_group_at(group, members, false, updated_at).await
|
||||
self.remove_members_from_group(group, members, false).await
|
||||
}
|
||||
|
||||
fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
|
||||
@@ -2377,19 +2235,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored `status` flag for a service-account status given on the admin
|
||||
/// or replication wire: the madmin `enabled` / `disabled` words and the stored
|
||||
/// `on` / `off` flags are both accepted; anything else disables the account.
|
||||
pub(crate) fn account_status_flag(status: &str) -> &'static str {
|
||||
match status {
|
||||
val if val == AccountStatus::Enabled.as_ref() => auth::ACCOUNT_ON,
|
||||
val if val == AccountStatus::Disabled.as_ref() => auth::ACCOUNT_OFF,
|
||||
auth::ACCOUNT_ON => auth::ACCOUNT_ON,
|
||||
auth::ACCOUNT_OFF => auth::ACCOUNT_OFF,
|
||||
_ => auth::ACCOUNT_OFF,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
|
||||
let default_policies = &DEFAULT_POLICIES;
|
||||
default_policies
|
||||
|
||||
+12
-285
@@ -385,14 +385,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
|
||||
self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_policy`] stamping the document with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock; see
|
||||
/// `IamCache::set_policy_at` (backlog#2291).
|
||||
pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
let updated_at = self.store.set_policy_at(name, policy, updated_at).await?;
|
||||
let updated_at = self.store.set_policy(name, policy).await?;
|
||||
|
||||
if !self.has_watcher() {
|
||||
for r in notify_iam_load_policy(name).await {
|
||||
@@ -650,18 +643,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result<OffsetDateTime> {
|
||||
self.set_user_status_at(name, status, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_user_status`] stamping the identity with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock (backlog#2291).
|
||||
pub async fn set_user_status_at(
|
||||
&self,
|
||||
name: &str,
|
||||
status: rustfs_madmin::AccountStatus,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let updated_at = self.store.set_user_status_at(name, status, updated_at).await?;
|
||||
let updated_at = self.store.set_user_status(name, status).await?;
|
||||
|
||||
self.notify_for_user(name, false).await;
|
||||
|
||||
@@ -673,20 +655,6 @@ impl<T: Store> IamSys<T> {
|
||||
parent_user: &str,
|
||||
groups: Option<Vec<String>>,
|
||||
opts: NewServiceAccountOpts,
|
||||
) -> Result<(Credentials, OffsetDateTime)> {
|
||||
self.new_service_account_at(parent_user, groups, opts, OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::new_service_account`] stamping the identity with `updated_at`
|
||||
/// (a replicated edit's source time) instead of the local clock
|
||||
/// (backlog#2291).
|
||||
pub async fn new_service_account_at(
|
||||
&self,
|
||||
parent_user: &str,
|
||||
groups: Option<Vec<String>>,
|
||||
opts: NewServiceAccountOpts,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<(Credentials, OffsetDateTime)> {
|
||||
if parent_user.is_empty() {
|
||||
return Err(IamError::InvalidArgument);
|
||||
@@ -756,18 +724,11 @@ impl<T: Store> IamSys<T> {
|
||||
let mut cred = create_new_credentials_with_metadata(&access_key, &secret_key, &m, &secret_key)?;
|
||||
cred.parent_user = parent_user.to_owned();
|
||||
cred.groups = groups;
|
||||
// The status is part of the created identity: a replicated disabled
|
||||
// account must never exist enabled, not even between a create and a
|
||||
// follow-up status write (backlog#2289).
|
||||
cred.status = opts
|
||||
.status
|
||||
.as_deref()
|
||||
.map_or(ACCOUNT_ON, crate::manager::account_status_flag)
|
||||
.to_owned();
|
||||
cred.status = ACCOUNT_ON.to_owned();
|
||||
cred.name = opts.name;
|
||||
cred.description = opts.description;
|
||||
|
||||
let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?;
|
||||
let create_at = self.store.add_service_account(cred.clone()).await?;
|
||||
|
||||
self.notify_for_service_account(&cred.access_key).await;
|
||||
|
||||
@@ -775,23 +736,11 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
|
||||
self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::update_service_account`] stamping the identity with
|
||||
/// `updated_at` (a replicated edit's source time) instead of the local
|
||||
/// clock (backlog#2291).
|
||||
pub async fn update_service_account_at(
|
||||
&self,
|
||||
name: &str,
|
||||
opts: UpdateServiceAccountOpts,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if name == SITE_REPLICATOR_SERVICE_ACCOUNT && !opts.allow_site_replicator_account {
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
|
||||
let updated_at = self.store.update_service_account_at(name, opts, updated_at).await?;
|
||||
let updated_at = self.store.update_service_account(name, opts).await?;
|
||||
|
||||
self.notify_for_service_account(name).await;
|
||||
|
||||
@@ -991,17 +940,6 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn create_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
|
||||
self.create_user_at(access_key, args, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::create_user`] stamping the identity with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock (backlog#2291).
|
||||
pub async fn create_user_at(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
}
|
||||
@@ -1014,7 +952,7 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::InvalidSecretKeyLength);
|
||||
}
|
||||
|
||||
let updated_at = self.store.add_user_at(access_key, args, updated_at).await?;
|
||||
let updated_at = self.store.add_user(access_key, args).await?;
|
||||
self.load_user(access_key, UserType::Reg).await?;
|
||||
|
||||
self.notify_for_user(access_key, false).await;
|
||||
@@ -1088,21 +1026,10 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn add_users_to_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
|
||||
self.add_users_to_group_at(group, users, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::add_users_to_group`] stamping the group with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock (backlog#2291).
|
||||
pub async fn add_users_to_group_at(
|
||||
&self,
|
||||
group: &str,
|
||||
users: Vec<String>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if contains_reserved_chars(group) {
|
||||
return Err(IamError::GroupNameContainsReservedChars);
|
||||
}
|
||||
let updated_at = self.store.add_users_to_group_at(group, users, updated_at).await?;
|
||||
let updated_at = self.store.add_users_to_group(group, users).await?;
|
||||
|
||||
self.notify_for_group(group).await;
|
||||
|
||||
@@ -1110,19 +1037,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn remove_users_from_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
|
||||
self.remove_users_from_group_at(group, users, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::remove_users_from_group`] stamping the group with `updated_at`
|
||||
/// (a replicated edit's source time) instead of the local clock
|
||||
/// (backlog#2291).
|
||||
pub async fn remove_users_from_group_at(
|
||||
&self,
|
||||
group: &str,
|
||||
users: Vec<String>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let updated_at = self.store.remove_users_from_group_at(group, users, updated_at).await?;
|
||||
let updated_at = self.store.remove_users_from_group(group, users).await?;
|
||||
|
||||
self.notify_for_group(group).await;
|
||||
|
||||
@@ -1130,13 +1045,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn set_group_status(&self, group: &str, enable: bool) -> Result<OffsetDateTime> {
|
||||
self.set_group_status_at(group, enable, OffsetDateTime::now_utc()).await
|
||||
}
|
||||
|
||||
/// [`Self::set_group_status`] stamping the group with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock (backlog#2291).
|
||||
pub async fn set_group_status_at(&self, group: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
let updated_at = self.store.set_group_status_at(group, enable, updated_at).await?;
|
||||
let updated_at = self.store.set_group_status(group, enable).await?;
|
||||
|
||||
self.notify_for_group(group).await;
|
||||
|
||||
@@ -1146,22 +1055,6 @@ impl<T: Store> IamSys<T> {
|
||||
self.store.get_group_description(group).await
|
||||
}
|
||||
|
||||
/// The stored group record itself (see `IamCache::get_group_info`).
|
||||
pub async fn get_group_info(&self, group: &str) -> Option<GroupInfo> {
|
||||
self.store.get_group_info(group).await
|
||||
}
|
||||
|
||||
/// The stored policy document, `Error::NoSuchPolicy` when absent.
|
||||
pub async fn get_policy_doc(&self, name: &str) -> Result<PolicyDoc> {
|
||||
self.store.get_policy_doc(name).await
|
||||
}
|
||||
|
||||
/// The stored mapping record for one user or group (see
|
||||
/// `IamCache::get_mapped_policy_record`).
|
||||
pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
|
||||
self.store.get_mapped_policy_record(name, user_type, is_group).await
|
||||
}
|
||||
|
||||
pub async fn list_groups_load(&self) -> Result<Vec<String>> {
|
||||
self.store.update_groups().await
|
||||
}
|
||||
@@ -1171,24 +1064,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
|
||||
self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`Self::policy_db_set`] stamping the mapping with `updated_at` (a
|
||||
/// replicated edit's source time) instead of the local clock (backlog#2291).
|
||||
pub async fn policy_db_set_at(
|
||||
&self,
|
||||
name: &str,
|
||||
user_type: UserType,
|
||||
is_group: bool,
|
||||
policy: &str,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let updated_at = self
|
||||
.store
|
||||
.policy_db_set_at(name, user_type, is_group, policy, updated_at)
|
||||
.await?;
|
||||
let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
|
||||
|
||||
if !self.has_watcher() {
|
||||
for r in notify_iam_load_policy_mapping(name, user_type.to_u64(), is_group).await {
|
||||
@@ -1970,11 +1846,6 @@ pub struct NewServiceAccountOpts {
|
||||
pub expiration: Option<OffsetDateTime>,
|
||||
pub allow_site_replicator_account: bool,
|
||||
pub claims: Option<HashMap<String, Value>>,
|
||||
/// Status the account is created with (`enabled` / `disabled` or the
|
||||
/// stored `on` / `off` flags); `None` creates it enabled. Site
|
||||
/// replication passes the source account's status so a disabled account
|
||||
/// is never enabled on the peer, not even transiently (backlog#2289).
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
pub struct UpdateServiceAccountOpts {
|
||||
@@ -2210,9 +2081,6 @@ mod tests {
|
||||
block_delete: Arc<std::sync::atomic::AtomicBool>,
|
||||
delete_started: Arc<tokio::sync::Notify>,
|
||||
release_delete: Arc<tokio::sync::Notify>,
|
||||
block_group_save: Arc<std::sync::atomic::AtomicBool>,
|
||||
group_save_started: Arc<tokio::sync::Notify>,
|
||||
group_save_release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
@@ -2226,9 +2094,6 @@ mod tests {
|
||||
block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
delete_started: Arc::new(tokio::sync::Notify::new()),
|
||||
release_delete: Arc::new(tokio::sync::Notify::new()),
|
||||
block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
group_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
group_save_release: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2332,15 +2197,11 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
|
||||
if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
self.group_save_started.notify_one();
|
||||
self.group_save_release.notified().await;
|
||||
}
|
||||
Ok(())
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn delete_group_info(&self, _name: &str) -> Result<()> {
|
||||
Ok(())
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()> {
|
||||
@@ -2517,140 +2378,6 @@ mod tests {
|
||||
IamSys::new(cache)
|
||||
}
|
||||
|
||||
async fn assert_group_write_during_reload_is_published(remove: bool) {
|
||||
let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
|
||||
let member = "sts-fallback-test-parent";
|
||||
let group = if remove { "testgroup" } else { "new-published-group" };
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
|
||||
iam_sys
|
||||
.store
|
||||
.api
|
||||
.block_group_save
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let before = iam_sys.store.cache.snapshot();
|
||||
let writer_iam = iam_sys.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
if remove {
|
||||
writer_iam
|
||||
.remove_users_from_group_at(group, vec![member.to_string()], source_time)
|
||||
.await
|
||||
} else {
|
||||
writer_iam
|
||||
.add_users_to_group_at(group, vec![member.to_string()], source_time)
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), iam_sys.store.api.group_save_started.notified())
|
||||
.await
|
||||
.expect("group save should reach the barrier");
|
||||
// The pending store write has not changed the cache, so the production
|
||||
// full-reload snapshot guard permits this replacement.
|
||||
assert!(iam_sys.store.cache.with_write_lock(|cache| cache.matches_snapshot(&before)));
|
||||
iam_sys
|
||||
.store
|
||||
.api
|
||||
.load_all(&iam_sys.store.cache)
|
||||
.await
|
||||
.expect("reload while group save is pending");
|
||||
iam_sys.store.api.group_save_release.notify_one();
|
||||
assert_eq!(writer.await.expect("join group writer").expect("group write should succeed"), source_time);
|
||||
let info = iam_sys
|
||||
.get_group_info(group)
|
||||
.await
|
||||
.expect("successful group write must remain readable after reload");
|
||||
assert_eq!(info.update_at, Some(source_time), "source timestamp must remain on the record");
|
||||
assert_eq!(info.members, if remove { Vec::new() } else { vec![member.to_string()] });
|
||||
let groups = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
|
||||
assert_eq!(
|
||||
groups.is_some_and(|groups| groups.contains(group)),
|
||||
!remove,
|
||||
"membership index must reflect the write"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn add_group_write_during_reload_publishes_after_store_save() {
|
||||
assert_group_write_during_reload_is_published(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn remove_group_write_during_reload_publishes_after_store_save() {
|
||||
assert_group_write_during_reload_is_published(true).await;
|
||||
}
|
||||
|
||||
/// Review finding on rustfs#7195: a replicated group edit carries a source
|
||||
/// stamp that may predate this node's cache load time. The stamp belongs on
|
||||
/// the record only; publishing the cache with it makes `LockedCache::exec`
|
||||
/// drop the write, so the group is written to the store but unreadable
|
||||
/// here and the receiver's next `set_group_status_at` fails with
|
||||
/// `NoSuchGroup`. Add, status and removal must all publish with the local
|
||||
/// clock while keeping the source stamp on `GroupInfo::update_at`.
|
||||
#[tokio::test]
|
||||
async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let member = "group-stamp-member";
|
||||
let identity = UserIdentity {
|
||||
version: 1,
|
||||
credentials: Credentials {
|
||||
access_key: member.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
update_at: Some(OffsetDateTime::now_utc()),
|
||||
};
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
|
||||
// The startup load publishes every entity with the load time.
|
||||
cache.replace_groups(CacheEntity::new(HashMap::new()));
|
||||
cache.replace_user_group_memberships(CacheEntity::new(HashMap::new()));
|
||||
});
|
||||
|
||||
let group = "group-stamp";
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
|
||||
let stamped = iam_sys
|
||||
.add_users_to_group_at(group, vec![member.to_string()], source_time)
|
||||
.await
|
||||
.expect("add members with a source stamp older than the cache load");
|
||||
assert_eq!(stamped, source_time, "the returned stamp is the source time");
|
||||
let info = iam_sys
|
||||
.get_group_info(group)
|
||||
.await
|
||||
.expect("the group must be readable right after the add");
|
||||
assert_eq!(info.members, vec![member.to_string()]);
|
||||
assert_eq!(info.update_at, Some(source_time), "the record keeps the source stamp");
|
||||
let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
|
||||
assert!(
|
||||
memberships.is_some_and(|groups| groups.contains(group)),
|
||||
"the membership index is published too"
|
||||
);
|
||||
|
||||
let disabled_at = source_time + time::Duration::seconds(1);
|
||||
iam_sys
|
||||
.set_group_status_at(group, false, disabled_at)
|
||||
.await
|
||||
.expect("status change with a source stamp older than the cache load");
|
||||
let info = iam_sys.get_group_info(group).await.expect("group after status change");
|
||||
assert_eq!(info.status, "disabled");
|
||||
assert_eq!(info.update_at, Some(disabled_at));
|
||||
|
||||
let removed_at = source_time + time::Duration::seconds(2);
|
||||
iam_sys
|
||||
.remove_users_from_group_at(group, vec![member.to_string()], removed_at)
|
||||
.await
|
||||
.expect("removal with a source stamp older than the cache load");
|
||||
let info = iam_sys.get_group_info(group).await.expect("group after removal");
|
||||
assert!(info.members.is_empty(), "the removal must be visible in the cache");
|
||||
assert_eq!(info.update_at, Some(removed_at));
|
||||
let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
|
||||
assert!(
|
||||
!memberships.is_some_and(|groups| groups.contains(group)),
|
||||
"the membership index follows the removal"
|
||||
);
|
||||
}
|
||||
|
||||
fn service_account_opts(access_key: &str, secret_key: &str) -> NewServiceAccountOpts {
|
||||
NewServiceAccountOpts {
|
||||
access_key: access_key.to_string(),
|
||||
|
||||
+11
-772
@@ -18,7 +18,7 @@ use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
|
||||
NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use time::macros::offset;
|
||||
use time::{self, Duration, OffsetDateTime};
|
||||
@@ -65,66 +65,6 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
|
||||
"Rule with ExpiredObjectDeleteMarker cannot have tags based filtering";
|
||||
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
|
||||
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
|
||||
const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer";
|
||||
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str =
|
||||
"Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And";
|
||||
const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates";
|
||||
const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key";
|
||||
const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters";
|
||||
const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative";
|
||||
const ERR_LIFECYCLE_FILTER_SIZE_RANGE: &str = "ObjectSizeGreaterThan must be smaller than ObjectSizeLessThan";
|
||||
/// Longest tag key S3 accepts.
|
||||
const MAX_TAG_KEY_LEN: usize = 128;
|
||||
/// Longest tag value S3 accepts.
|
||||
const MAX_TAG_VALUE_LEN: usize = 256;
|
||||
|
||||
/// A validation failure that the S3 boundary must answer with `MalformedXML`
|
||||
/// rather than `InvalidArgument`: the document does not match the published
|
||||
/// schema shape (wrong number of `Filter` predicates, a one-member `And`).
|
||||
///
|
||||
/// Everything else stays [`std::io::ErrorKind::Other`], which the boundary
|
||||
/// already maps to `InvalidArgument`.
|
||||
pub const LIFECYCLE_MALFORMED_XML_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
|
||||
|
||||
/// A persisted rule that could never have passed validation. Callers that can
|
||||
/// report an error surface it; evaluation itself stays fail-closed and takes
|
||||
/// no action for the rule.
|
||||
pub const LIFECYCLE_CORRUPT_RULE_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
|
||||
|
||||
fn malformed_xml_error(message: &'static str) -> std::io::Error {
|
||||
std::io::Error::new(LIFECYCLE_MALFORMED_XML_ERROR_KIND, message)
|
||||
}
|
||||
|
||||
/// The retention count a rule keeps, or `None` when the persisted value is
|
||||
/// negative — a shape PUT validation rejects, so reaching it means the rule
|
||||
/// came from older persistence or an import.
|
||||
///
|
||||
/// A negative count must never be read as "retain everything": that is how an
|
||||
/// invalid configuration silently stopped deleting versions (backlog#2201).
|
||||
pub fn retained_noncurrent_versions(count: i32) -> Option<usize> {
|
||||
usize::try_from(count).ok()
|
||||
}
|
||||
|
||||
/// Does any rule carry a retention count that validation would have rejected?
|
||||
pub fn lifecycle_has_corrupt_retention_count(lc: &BucketLifecycleConfiguration) -> bool {
|
||||
lc.rules.iter().any(rule_has_corrupt_retention_count)
|
||||
}
|
||||
|
||||
fn rule_has_corrupt_retention_count(rule: &LifecycleRule) -> bool {
|
||||
let expiration_count = rule
|
||||
.noncurrent_version_expiration
|
||||
.as_ref()
|
||||
.and_then(|expiration| expiration.newer_noncurrent_versions);
|
||||
let transition_counts = rule
|
||||
.noncurrent_version_transitions
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter_map(|transition| transition.newer_noncurrent_versions);
|
||||
expiration_count
|
||||
.into_iter()
|
||||
.chain(transition_counts)
|
||||
.any(|count| retained_noncurrent_versions(count).is_none())
|
||||
}
|
||||
|
||||
pub use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
|
||||
@@ -201,17 +141,6 @@ impl RuleValidate for LifecycleRule {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT));
|
||||
}
|
||||
|
||||
if let Some(filter) = self.filter.as_ref() {
|
||||
validate_lifecycle_filter(filter)?;
|
||||
}
|
||||
|
||||
// A negative retention count was accepted and then read as "retain
|
||||
// (almost) everything" during evaluation, so an HTTP-accepted rule
|
||||
// silently stopped deleting versions (backlog#2201).
|
||||
if rule_has_corrupt_retention_count(self) {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS));
|
||||
}
|
||||
|
||||
// Rule with DelMarkerExpiration cannot have tags based filtering
|
||||
let has_tag_filter = self
|
||||
.filter
|
||||
@@ -244,14 +173,11 @@ impl RuleValidate for LifecycleRule {
|
||||
// Rule must have at least one action
|
||||
let has_expiration = self.expiration.is_some();
|
||||
let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
|
||||
// `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS
|
||||
// form: it keeps the newest N noncurrent versions and expires the rest
|
||||
// with no age condition. RustFS accepts it for MinIO compatibility, so
|
||||
// it has to count as an action here — otherwise a count-only rule was
|
||||
// rejected as actionless (backlog#2201).
|
||||
let has_noncurrent_expiration = self.noncurrent_version_expiration.as_ref().is_some_and(|expiration| {
|
||||
expiration.noncurrent_days.is_some() || expiration.newer_noncurrent_versions.is_some_and(|count| count > 0)
|
||||
});
|
||||
let has_noncurrent_expiration = self
|
||||
.noncurrent_version_expiration
|
||||
.as_ref()
|
||||
.and_then(|e| e.noncurrent_days)
|
||||
.is_some();
|
||||
let has_noncurrent_transition = self
|
||||
.noncurrent_version_transitions
|
||||
.as_ref()
|
||||
@@ -277,81 +203,6 @@ impl RuleValidate for LifecycleRule {
|
||||
}
|
||||
}
|
||||
|
||||
/// Structural validation for `LifecycleRuleFilter`.
|
||||
///
|
||||
/// The generated DTO is all-`Option`, so the S3 schema constraints have to be
|
||||
/// checked here: at most one top-level predicate, an `And` that actually
|
||||
/// combines at least two, no repeated tag key, tag key/value limits, and a
|
||||
/// coherent non-negative size range (backlog#2201).
|
||||
///
|
||||
/// A filter with no predicate at all stays valid: AWS documents an empty
|
||||
/// `Filter` as "applies to every object in the bucket", and rejecting it would
|
||||
/// break the most common way to write an unconditional rule.
|
||||
fn validate_lifecycle_filter(filter: &LifecycleRuleFilter) -> Result<(), std::io::Error> {
|
||||
let top_level_predicates = usize::from(filter.prefix.is_some())
|
||||
+ usize::from(filter.tag.is_some())
|
||||
+ usize::from(filter.object_size_greater_than.is_some())
|
||||
+ usize::from(filter.object_size_less_than.is_some())
|
||||
+ usize::from(filter.and.is_some());
|
||||
if top_level_predicates > 1 {
|
||||
return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES));
|
||||
}
|
||||
|
||||
if let Some(tag) = filter.tag.as_ref() {
|
||||
validate_lifecycle_tag(tag)?;
|
||||
}
|
||||
|
||||
if let Some(and) = filter.and.as_ref() {
|
||||
let tags = and.tags.as_deref().unwrap_or(&[]);
|
||||
let and_predicates = usize::from(and.prefix.is_some())
|
||||
+ tags.len()
|
||||
+ usize::from(and.object_size_greater_than.is_some())
|
||||
+ usize::from(and.object_size_less_than.is_some());
|
||||
if and_predicates < 2 {
|
||||
return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES));
|
||||
}
|
||||
let mut seen_keys = HashSet::with_capacity(tags.len());
|
||||
for tag in tags {
|
||||
validate_lifecycle_tag(tag)?;
|
||||
let key = tag.key.as_deref().unwrap_or_default();
|
||||
if !seen_keys.insert(key) {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY));
|
||||
}
|
||||
}
|
||||
validate_lifecycle_size_bounds(and.object_size_greater_than, and.object_size_less_than)?;
|
||||
}
|
||||
|
||||
validate_lifecycle_size_bounds(filter.object_size_greater_than, filter.object_size_less_than)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// S3 requires a tag to carry a key and value; both are length-bounded.
|
||||
/// The DTO makes both optional, so incomplete tags have to be rejected here
|
||||
/// rather than silently matching nothing.
|
||||
fn validate_lifecycle_tag(tag: &s3s::dto::Tag) -> Result<(), std::io::Error> {
|
||||
let key = tag.key.as_deref().unwrap_or_default();
|
||||
let Some(value) = tag.value.as_deref() else {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
|
||||
};
|
||||
if key.is_empty() || key.chars().count() > MAX_TAG_KEY_LEN || value.chars().count() > MAX_TAG_VALUE_LEN {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_lifecycle_size_bounds(greater_than: Option<i64>, less_than: Option<i64>) -> Result<(), std::io::Error> {
|
||||
if greater_than.is_some_and(|size| size < 0) || less_than.is_some_and(|size| size < 0) {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE));
|
||||
}
|
||||
if let (Some(greater_than), Some(less_than)) = (greater_than, less_than)
|
||||
&& greater_than >= less_than
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_SIZE_RANGE));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> {
|
||||
// Prefer a non-empty legacy prefix; treat an empty legacy prefix as if it were not set
|
||||
if let Some(p) = rule.prefix.as_deref()
|
||||
@@ -442,10 +293,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// A positive count is an action on its own (the MinIO count-only
|
||||
// form). Zero means "no count constraint" here, exactly as the
|
||||
// batch limit path reads it, and a negative count is corrupt —
|
||||
// neither makes the rule active (backlog#2201).
|
||||
if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions
|
||||
&& newer_noncurrent_versions > 0
|
||||
{
|
||||
@@ -716,23 +563,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
|
||||
if let Some(ref lc_rules) = self.filter_rules(obj).await {
|
||||
for rule in lc_rules.iter() {
|
||||
// A retention count that PUT validation would have rejected can
|
||||
// only come from older persistence or an import. Take no action
|
||||
// for the rule instead of allowing another action on the same
|
||||
// corrupt rule to delete or transition an object (backlog#2201).
|
||||
if rule_has_corrupt_retention_count(rule) {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
object = %obj.name,
|
||||
rule_id = %rule.id.clone().unwrap_or_default(),
|
||||
reason = "corrupt_newer_noncurrent_versions",
|
||||
"Skipped lifecycle evaluation for a rule with an invalid retention count"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if obj.is_latest && obj.expired_object_deletemarker() {
|
||||
if let Some(expiration) = rule.expiration.as_ref()
|
||||
&& expiration.expired_object_delete_marker.is_some_and(|v| v)
|
||||
@@ -789,18 +619,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
|
||||
if !obj.is_latest
|
||||
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
|
||||
&& (noncurrent_version_expiration.noncurrent_days.is_some()
|
||||
|| noncurrent_version_expiration
|
||||
.newer_noncurrent_versions
|
||||
.is_some_and(|count| count > 0))
|
||||
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
|
||||
&& noncurrent_version_expiration
|
||||
.newer_noncurrent_versions
|
||||
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
|
||||
{
|
||||
// A count-only rule (MinIO extension) has no age condition:
|
||||
// every version past the retained count is due as soon as it
|
||||
// became noncurrent, i.e. zero days after the successor.
|
||||
let noncurrent_days = noncurrent_version_expiration.noncurrent_days.unwrap_or(0);
|
||||
if let Some(successor_mod_time) = obj.successor_mod_time {
|
||||
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
|
||||
if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
|
||||
@@ -968,18 +791,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
for rule in filter_rules.iter() {
|
||||
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
|
||||
return if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
|
||||
// Zero means "no count constraint"; a negative count is
|
||||
// corrupt and must not be read as "retain everything"
|
||||
// (backlog#2201). Neither yields a limit event.
|
||||
let Some(retained) = retained_noncurrent_versions(newer_noncurrent_versions).filter(|c| *c > 0) else {
|
||||
if newer_noncurrent_versions == 0 {
|
||||
continue;
|
||||
};
|
||||
}
|
||||
Event {
|
||||
action: IlmAction::DeleteVersionAction,
|
||||
rule_id: rule.id.clone().unwrap_or_default(),
|
||||
noncurrent_days: u32::try_from(noncurrent_version_expiration.noncurrent_days.unwrap_or(0))
|
||||
.unwrap_or(u32::MAX),
|
||||
newer_noncurrent_versions: retained,
|
||||
newer_noncurrent_versions: usize::try_from(newer_noncurrent_versions).unwrap_or(usize::MAX),
|
||||
due: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
storage_class: "".into(),
|
||||
}
|
||||
@@ -1342,11 +1162,7 @@ mod tests {
|
||||
use super::*;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use s3s::dto::{
|
||||
LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, NoncurrentVersionTransition,
|
||||
TransitionStorageClass,
|
||||
};
|
||||
use s3s::xml::{Deserialize as XmlDeserialize, SerializeContent as XmlSerializeContent};
|
||||
use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use time::macros::datetime;
|
||||
@@ -4367,583 +4183,6 @@ mod tests {
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
// ---- backlog#2201: retention-count and Filter invariants -----------------
|
||||
|
||||
fn rule_with_noncurrent_expiration(expiration: NoncurrentVersionExpiration) -> LifecycleRule {
|
||||
LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("noncurrent".to_string()),
|
||||
noncurrent_version_expiration: Some(expiration),
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn rule_with_filter(filter: LifecycleRuleFilter) -> LifecycleRule {
|
||||
LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: Some(filter),
|
||||
id: Some("filtered".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn config_with_rules(rules: Vec<LifecycleRule>) -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules,
|
||||
}
|
||||
}
|
||||
|
||||
fn tag(key: &str, value: &str) -> s3s::dto::Tag {
|
||||
s3s::dto::Tag {
|
||||
key: Some(key.to_string()),
|
||||
value: Some(value.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_negative_newer_noncurrent_versions() {
|
||||
// A negative retention count used to be accepted and then read as
|
||||
// usize::MAX during evaluation, so the rule silently stopped deleting
|
||||
// versions (backlog#2201).
|
||||
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(30),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
})]);
|
||||
|
||||
let err = lc
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect_err("a negative retention count must be rejected");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS);
|
||||
assert_ne!(err.kind(), LIFECYCLE_MALFORMED_XML_ERROR_KIND, "value errors stay InvalidArgument");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_negative_newer_noncurrent_versions_on_transition() {
|
||||
let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(30),
|
||||
newer_noncurrent_versions: None,
|
||||
});
|
||||
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
|
||||
newer_noncurrent_versions: Some(-3),
|
||||
noncurrent_days: Some(1),
|
||||
storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
|
||||
}]);
|
||||
|
||||
// The transition validator already refuses a negative count, and it runs
|
||||
// first, so this pins the rejection rather than the message. The gap
|
||||
// this PR closes is the expiration side, which had no such check.
|
||||
config_with_rules(vec![rule])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect_err("a negative retention count on a transition must be rejected");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_newer_noncurrent_versions_means_no_count_constraint() {
|
||||
// Zero carries no constraint, matching how the batch limit path has
|
||||
// always read it. Alongside an age condition the rule is valid; on its
|
||||
// own it says nothing, so the rule has no action.
|
||||
config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(30),
|
||||
newer_noncurrent_versions: Some(0),
|
||||
})])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("zero count alongside NoncurrentDays is valid");
|
||||
|
||||
let err = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(0),
|
||||
})])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect_err("a zero count on its own is not an action");
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_accepts_count_only_noncurrent_expiration() {
|
||||
// MinIO extension: NewerNoncurrentVersions with no NoncurrentDays. It
|
||||
// used to be rejected as an actionless rule (backlog#2201).
|
||||
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(2),
|
||||
})]);
|
||||
|
||||
lc.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("a count-only noncurrent expiration rule is accepted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn eval_inner_expires_versions_beyond_count_only_retention() {
|
||||
// Count-only rules have no age condition: everything past the retained
|
||||
// count is due as soon as it became noncurrent.
|
||||
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(2),
|
||||
})]);
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
|
||||
successor_mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
|
||||
is_latest: false,
|
||||
num_versions: 5,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Rank 2 is the third-newest noncurrent version: past a retention of 2.
|
||||
let expired = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 2).await;
|
||||
assert_eq!(expired.action, IlmAction::DeleteVersionAction);
|
||||
assert_eq!(expired.rule_id, "noncurrent");
|
||||
|
||||
// Rank 1 is still within the retained count.
|
||||
let retained = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 1).await;
|
||||
assert_eq!(retained.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_keeps_age_condition_when_count_and_days_are_set() {
|
||||
// With both set, the count gates which versions are candidates and the
|
||||
// age condition still decides when they are due.
|
||||
with_default_ilm_process_time(|| {});
|
||||
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(10),
|
||||
newer_noncurrent_versions: Some(1),
|
||||
})]);
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
is_latest: false,
|
||||
num_versions: 3,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let too_young = lc.eval_inner(&opts, datetime!(2025-01-05 00:00:00 UTC), 2).await;
|
||||
assert_eq!(too_young.action, IlmAction::NoneAction, "the age condition still applies");
|
||||
|
||||
let due = lc.eval_inner(&opts, datetime!(2025-01-20 00:00:00 UTC), 2).await;
|
||||
assert_eq!(due.action, IlmAction::DeleteVersionAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn eval_inner_takes_no_action_for_a_corrupt_retention_count() {
|
||||
// Reachable only from older persistence or an import; it must not be
|
||||
// read as "retain everything", and it must not delete either.
|
||||
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
})]);
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
is_latest: false,
|
||||
num_versions: 3,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 2).await;
|
||||
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn eval_inner_does_not_expire_latest_object_for_a_corrupt_retention_rule() {
|
||||
let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
});
|
||||
rule.expiration = Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
});
|
||||
let lc = config_with_rules(vec![rule]);
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
is_latest: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
|
||||
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn eval_inner_does_not_delete_latest_marker_for_a_corrupt_retention_rule() {
|
||||
let mut expired_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
});
|
||||
expired_marker_rule.expiration = Some(LifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut aged_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
});
|
||||
aged_marker_rule.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(1) });
|
||||
|
||||
for rule in [expired_marker_rule, aged_marker_rule] {
|
||||
let lc = config_with_rules(vec![rule]);
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
is_latest: true,
|
||||
delete_marker: true,
|
||||
num_versions: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
|
||||
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_retention_count_is_detected_on_either_action() {
|
||||
let mut transition_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(0),
|
||||
});
|
||||
transition_rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
noncurrent_days: Some(1),
|
||||
storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
|
||||
}]);
|
||||
|
||||
assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
|
||||
rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
})
|
||||
])));
|
||||
assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![transition_rule])));
|
||||
assert!(!lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
|
||||
rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(3),
|
||||
})
|
||||
])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_only_rules_are_active_only_for_a_positive_count() {
|
||||
let positive = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(2),
|
||||
})]);
|
||||
assert!(positive.has_active_rules(""));
|
||||
|
||||
let corrupt = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
})]);
|
||||
assert!(!corrupt.has_active_rules(""), "a corrupt retention count must not make a rule active");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn noncurrent_versions_expiration_limit_ignores_a_corrupt_count() {
|
||||
// The batch path must not read a negative count as "retain everything".
|
||||
let lc = Arc::new(config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
})]));
|
||||
let opts = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
|
||||
is_latest: false,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event = lc.noncurrent_versions_expiration_limit(&opts).await;
|
||||
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
assert_eq!(event.newer_noncurrent_versions, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_covers_filter_invariants() {
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
filter: LifecycleRuleFilter,
|
||||
expected: Option<(&'static str, std::io::ErrorKind)>,
|
||||
}
|
||||
|
||||
let cases = vec![
|
||||
Case {
|
||||
// AWS documents an empty Filter as "every object in the bucket".
|
||||
name: "empty filter applies to all objects",
|
||||
filter: LifecycleRuleFilter::default(),
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "single prefix predicate",
|
||||
filter: LifecycleRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "two top-level predicates",
|
||||
filter: LifecycleRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
tag: Some(tag("env", "prod")),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
|
||||
},
|
||||
Case {
|
||||
name: "prefix alongside And",
|
||||
filter: LifecycleRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
prefix: Some("logs/".to_string()),
|
||||
tags: Some(vec![tag("env", "prod")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
|
||||
},
|
||||
Case {
|
||||
name: "And with a single member",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
prefix: Some("logs/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
|
||||
},
|
||||
Case {
|
||||
name: "And with two members",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
prefix: Some("logs/".to_string()),
|
||||
tags: Some(vec![tag("env", "prod")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "And with two tags",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
tags: Some(vec![tag("env", "prod"), tag("team", "storage")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "And repeating a tag key",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
tags: Some(vec![tag("env", "prod"), tag("env", "dev")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "empty tag key",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(tag("", "prod")),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "missing tag key",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(s3s::dto::Tag {
|
||||
key: None,
|
||||
value: Some("prod".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "missing tag value",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(s3s::dto::Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: None,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "empty tag value",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(tag("env", "")),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "tag key at the limit",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN), "prod")),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
Case {
|
||||
name: "tag key past the limit",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN + 1), "prod")),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "tag value past the limit",
|
||||
filter: LifecycleRuleFilter {
|
||||
tag: Some(tag("env", &"v".repeat(MAX_TAG_VALUE_LEN + 1))),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "negative ObjectSizeGreaterThan",
|
||||
filter: LifecycleRuleFilter {
|
||||
object_size_greater_than: Some(-1),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "negative ObjectSizeLessThan",
|
||||
filter: LifecycleRuleFilter {
|
||||
object_size_less_than: Some(-5),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "inverted size range inside And",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
object_size_greater_than: Some(100),
|
||||
object_size_less_than: Some(100),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: Some((ERR_LIFECYCLE_FILTER_SIZE_RANGE, std::io::ErrorKind::Other)),
|
||||
},
|
||||
Case {
|
||||
name: "valid size range inside And",
|
||||
filter: LifecycleRuleFilter {
|
||||
and: Some(LifecycleRuleAndOperator {
|
||||
object_size_greater_than: Some(1),
|
||||
object_size_less_than: Some(2),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
expected: None,
|
||||
},
|
||||
];
|
||||
|
||||
for case in cases {
|
||||
let result = config_with_rules(vec![rule_with_filter(case.filter)])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await;
|
||||
match (case.expected, result) {
|
||||
(None, Ok(())) => {}
|
||||
(None, Err(err)) => panic!("{}: expected acceptance, got {err}", case.name),
|
||||
(Some((message, _)), Ok(())) => panic!("{}: expected rejection with {message}", case.name),
|
||||
(Some((message, kind)), Err(err)) => {
|
||||
assert_eq!(err.to_string(), message, "{}", case.name);
|
||||
assert_eq!(err.kind(), kind, "{}: wrong S3 error category", case.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_keeps_legacy_prefix_and_filter_mutually_exclusive() {
|
||||
let mut rule = rule_with_filter(LifecycleRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
rule.prefix = Some("legacy/".to_string());
|
||||
|
||||
let err = config_with_rules(vec![rule])
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect_err("legacy Prefix and Filter cannot both be present");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_only_rule_round_trips_through_xml() {
|
||||
// The MinIO count-only form has to survive the wire codec, or the rule
|
||||
// this PR now accepts could not be persisted and read back.
|
||||
let xml = br#"<LifecycleConfiguration><Rule><ID>count-only</ID><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>2</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
|
||||
let mut deserializer = s3s::xml::Deserializer::new(xml);
|
||||
let parsed =
|
||||
<BucketLifecycleConfiguration as XmlDeserialize>::deserialize(&mut deserializer).expect("count-only XML parses");
|
||||
|
||||
let expiration = parsed.rules[0]
|
||||
.noncurrent_version_expiration
|
||||
.as_ref()
|
||||
.expect("noncurrent expiration is present");
|
||||
assert_eq!(expiration.newer_noncurrent_versions, Some(2));
|
||||
assert_eq!(expiration.noncurrent_days, None);
|
||||
|
||||
let mut buf = Vec::new();
|
||||
let mut serializer = s3s::xml::Serializer::new(&mut buf);
|
||||
XmlSerializeContent::serialize_content(&parsed, &mut serializer).expect("count-only config serializes");
|
||||
let serialized = String::from_utf8(buf).expect("serialized XML is UTF-8");
|
||||
assert!(
|
||||
serialized.contains("<NewerNoncurrentVersions>2</NewerNoncurrentVersions>"),
|
||||
"retention count survives the round trip: {serialized}"
|
||||
);
|
||||
assert!(
|
||||
!serialized.contains("<NoncurrentDays>"),
|
||||
"a count-only rule must not gain an age condition: {serialized}"
|
||||
);
|
||||
}
|
||||
|
||||
mod adversarial_regressions {
|
||||
use super::*;
|
||||
use s3s::dto::NoncurrentVersionExpiration;
|
||||
|
||||
@@ -22,10 +22,7 @@ use rustfs_replication::ReplicationStatusType;
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
|
||||
use crate::object_lock;
|
||||
use crate::{
|
||||
Event, LIFECYCLE_CORRUPT_RULE_ERROR_KIND, Lifecycle, ObjectOpts, expiration_action_has_valid_target,
|
||||
lifecycle_has_corrupt_retention_count,
|
||||
};
|
||||
use crate::{Event, Lifecycle, ObjectOpts, expiration_action_has_valid_target};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
@@ -155,17 +152,6 @@ impl Evaluator {
|
||||
format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()),
|
||||
));
|
||||
}
|
||||
// PUT validation rejects a negative retention count, so a rule that
|
||||
// carries one came from older persistence or an import. Report it
|
||||
// instead of evaluating a configuration that cannot be honoured;
|
||||
// `eval_inner` independently takes no action for such a rule
|
||||
// (backlog#2201).
|
||||
if lifecycle_has_corrupt_retention_count(&self.policy) {
|
||||
return Err(std::io::Error::new(
|
||||
LIFECYCLE_CORRUPT_RULE_ERROR_KIND,
|
||||
"lifecycle configuration carries a negative 'NewerNoncurrentVersions'",
|
||||
));
|
||||
}
|
||||
Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await)
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user