mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 04:25:54 +00:00
Compare commits
56 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| df06b7e168 | |||
| 672087ec0d | |||
| 9dbeae1b45 | |||
| 6018dd372f | |||
| d633a635ec | |||
| fbd30a6f43 | |||
| a32360198c | |||
| 05efc584b0 | |||
| a70c96d520 | |||
| b0f68f5d0a | |||
| df554439b0 | |||
| 3b404e56c0 | |||
| ff00872922 | |||
| 4d1ce9618a | |||
| a2bad0953f | |||
| 14c99a994c | |||
| f6c2a9bfe0 | |||
| 640d7e0e3c | |||
| 409ac3de66 | |||
| 2a63fcbea6 | |||
| f4049598e4 | |||
| 1651541d38 | |||
| 85849788af | |||
| 975983abdd | |||
| cf1c45eb91 | |||
| 536c283716 | |||
| a04ddc237e | |||
| 7de6ac82e1 | |||
| 7993e11058 | |||
| 37ead2f69c | |||
| 1b1e590df7 | |||
| 086ee8e48a | |||
| 22bff27aee | |||
| 8a20498705 | |||
| c3e6d90c6f | |||
| d2af8f073e | |||
| 2baba1bda7 | |||
| bf2ca9113f | |||
| f474ea30d4 | |||
| 002ac9544c | |||
| d74d970e24 | |||
| cc15eae479 | |||
| c130d00d4b | |||
| e49d9cdea2 | |||
| f5778d8b97 | |||
| 9da42a899a | |||
| 3d46ed312a | |||
| 3496277e7c | |||
| 0b72f39023 | |||
| 30a0937a7d | |||
| 5b962b6c58 | |||
| 0ee5408b94 | |||
| cb3100a252 | |||
| 3a4afe9b38 | |||
| 71859ff83c | |||
| cb72df269a |
@@ -582,59 +582,13 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: |
|
||||
python3 - <<'PYBUILD'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(["git", *args], text=True).strip()
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with pathlib.Path(path).open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"]
|
||||
commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
|
||||
clean_before = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_before:
|
||||
raise SystemExit("hooks binary requires a clean build checkout")
|
||||
lock_sha256 = sha256("Cargo.lock")
|
||||
lock_git_blob = git("hash-object", "Cargo.lock")
|
||||
rustc = subprocess.check_output(["rustc", "-vV"], text=True)
|
||||
host = next(line.removeprefix("host: ") for line in rustc.splitlines() if line.startswith("host: "))
|
||||
if os.environ.get("CARGO_BUILD_TARGET") or pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target")).resolve() != pathlib.Path("target").resolve():
|
||||
raise SystemExit("this artifact requires the native target/debug output")
|
||||
subprocess.run(argv, check=True)
|
||||
clean_after = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_after or commit != git("rev-parse", "HEAD") or tree != git("rev-parse", "HEAD^{tree}") or lock_sha256 != sha256("Cargo.lock"):
|
||||
raise SystemExit("hooks binary source changed while building")
|
||||
manifest = {
|
||||
"schema": 1, "commit": commit, "tree": tree,
|
||||
"clean_before": clean_before, "clean_after": clean_after,
|
||||
"lock_sha256": lock_sha256, "lock_git_blob": lock_git_blob,
|
||||
"argv": argv, "profile": "debug", "target": host,
|
||||
"features": ["e2e-test-hooks"],
|
||||
"rustc_verbose": rustc,
|
||||
"build_flags": {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_TARGET_DIR", "RUSTUP_TOOLCHAIN") if key in os.environ},
|
||||
"binary_sha256": sha256("target/debug/rustfs"),
|
||||
}
|
||||
pathlib.Path("target/debug/rustfs.e2e-startup-cas-build.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
PYBUILD
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e-startup-cas-build.json
|
||||
path: target/debug/rustfs
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -900,6 +854,12 @@ jobs:
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
# a skip instead of failing, so surface that here where it is visible.
|
||||
if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
|
||||
echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
@@ -952,36 +912,6 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Preserve startup CAS binary input
|
||||
env:
|
||||
STARTUP_CAS_INPUT: ${{ runner.temp }}/rustfs-startup-cas-input
|
||||
run: |
|
||||
python3 - <<'PYINPUT'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
source = pathlib.Path("target/debug/rustfs")
|
||||
manifest_path = source.with_name("rustfs.e2e-startup-cas-build.json")
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
target = pathlib.Path(os.environ["STARTUP_CAS_INPUT"])
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
binary = target / "rustfs"
|
||||
shutil.copy2(source, binary)
|
||||
digest = hashlib.sha256()
|
||||
with binary.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
|
||||
raise SystemExit("downloaded hooks binary identity mismatch")
|
||||
shutil.copy2(manifest_path, target / manifest_path.name)
|
||||
binary.chmod(0o755)
|
||||
PYINPUT
|
||||
|
||||
- name: Verify e2e full membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
@@ -994,10 +924,6 @@ jobs:
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
run: cargo nextest run --profile e2e-full -p e2e_test
|
||||
|
||||
- name: Upload junit
|
||||
@@ -1010,17 +936,6 @@ jobs:
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: fresh-startup-cas-evidence-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
|
||||
@@ -121,6 +121,22 @@ jobs:
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
# Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
|
||||
# prerelease (alpha/beta/rc) also moves `latest`, so users pulling
|
||||
# `latest` get the newest test build. Once a stable tag is published
|
||||
# this returns false and `latest` follows stable releases only.
|
||||
prerelease_moves_latest() {
|
||||
local stable_tags
|
||||
stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
|
||||
| awk '{print $2}' \
|
||||
| grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
|
||||
if [[ -z "$stable_tags" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "ℹ️ Stable release tag(s) already exist; prereleases no longer update latest"
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
echo "🔗 Triggered by build workflow completion"
|
||||
@@ -184,8 +200,8 @@ jobs:
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
create_latest=true
|
||||
echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
|
||||
else
|
||||
@@ -243,8 +259,8 @@ jobs:
|
||||
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
create_latest=true
|
||||
echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
|
||||
else
|
||||
@@ -394,11 +410,13 @@ jobs:
|
||||
TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
|
||||
TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
|
||||
|
||||
# Add channel tags for prereleases and latest for stable
|
||||
# Add latest when requested (stable releases, and prereleases before GA)
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
|
||||
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
fi
|
||||
|
||||
# Always add the channel tag for prereleases, independent of latest
|
||||
if [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
# Prerelease channel tags (alpha, beta, rc)
|
||||
if [[ "$VERSION" == *"alpha"* ]]; then
|
||||
CHANNEL="alpha"
|
||||
@@ -555,7 +573,7 @@ jobs:
|
||||
"prerelease")
|
||||
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
|
||||
echo "⚠️ This is a prerelease image - use with caution"
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
# Prereleases move latest until the first stable tag exists (pre-GA policy).
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
echo "🏷️ Latest tag has been created for prerelease: $VERSION"
|
||||
else
|
||||
|
||||
Generated
+38
-38
@@ -2527,18 +2527,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.16"
|
||||
version = "0.5.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
|
||||
checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.7"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
@@ -2546,27 +2546,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.20"
|
||||
version = "0.9.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.13"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
|
||||
checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.22"
|
||||
version = "0.8.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
@@ -3673,9 +3673,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"pem-rfc7468 1.0.0",
|
||||
@@ -3946,7 +3946,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users 0.5.2",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4057,7 +4057,6 @@ dependencies = [
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"suppaftp",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -4091,7 +4090,7 @@ version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve 0.14.1",
|
||||
"rfc6979 0.6.0",
|
||||
@@ -4296,7 +4295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5694,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
|
||||
dependencies = [
|
||||
"io-lifetimes 3.0.1",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5735,9 +5734,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.12.1"
|
||||
version = "2.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
|
||||
checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -5759,7 +5758,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6140,9 +6139,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
version = "2.3.1"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
|
||||
checksum = "561a8da1a50e1428d3c51321dafeca849df992a5bb67720c386131234caba82e"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"crc32fast",
|
||||
@@ -6956,7 +6955,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7934,7 +7933,7 @@ version = "0.8.0-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"spki 0.8.0",
|
||||
]
|
||||
|
||||
@@ -7977,7 +7976,7 @@ dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand_core 0.10.1",
|
||||
"scrypt 0.12.0",
|
||||
@@ -8001,7 +8000,7 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"pkcs5 0.8.1",
|
||||
"rand_core 0.10.1",
|
||||
"spki 0.8.0",
|
||||
@@ -8707,7 +8706,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8943,9 +8942,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
||||
checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -9327,7 +9326,7 @@ dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"digest 0.11.3",
|
||||
"ecdsa 0.17.0",
|
||||
"ed25519-dalek 3.0.0",
|
||||
@@ -10746,6 +10745,7 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
"rustfs-lock",
|
||||
@@ -10943,9 +10943,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-uring"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0486e62d0efe25db95c00aeacb2da84368adcba299216cda99fcb11328061c84"
|
||||
checksum = "b29bc57b4bd62a73f4fae408b536adf578332e50e464797d09dc2382c7cb68c2"
|
||||
dependencies = [
|
||||
"io-uring",
|
||||
"libc",
|
||||
@@ -11066,7 +11066,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11149,7 +11149,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11420,7 +11420,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"ctutils",
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11994,7 +11994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.1",
|
||||
"der 0.8.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12409,7 +12409,7 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13528,7 +13528,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+5
-5
@@ -256,10 +256,10 @@ clap = { version = "4.6.6" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.0.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
@@ -306,7 +306,7 @@ rustfs-erasure-codec = { version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
|
||||
@@ -422,9 +422,9 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
/// Legacy, unverified repair notice. Its identity lacks kind, set scope,
|
||||
/// bucket incarnation and responsibility generation. Consumers must not use
|
||||
/// it to discharge persisted repair responsibility.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
@@ -439,8 +439,8 @@ const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
/// Record a legacy notification for compatibility. This is not an
|
||||
/// acknowledgement of storage verification or durable repair completion.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
@@ -515,6 +515,9 @@ mod tests {
|
||||
}
|
||||
coalescer_release(&key, Some(lease));
|
||||
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
|
||||
assert_ne!(lease, retry_lease);
|
||||
coalescer_release(&key, Some(lease));
|
||||
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
|
||||
coalescer_release(&key, Some(retry_lease));
|
||||
}
|
||||
|
||||
|
||||
@@ -144,6 +144,3 @@ russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -184,6 +184,8 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,8 @@ mod tests {
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::info;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tracing::warn;
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
@@ -115,6 +117,49 @@ mod tests {
|
||||
}
|
||||
|
||||
impl TcpPortBlackhole {
|
||||
/// Environment flag that turns an unusable fault-injection host into a
|
||||
/// hard failure instead of a logged skip. Lanes that provision
|
||||
/// `CAP_NET_ADMIN` set it so a broken runner cannot pass silently.
|
||||
#[cfg(target_os = "linux")]
|
||||
const REQUIRE_ENV: &str = "RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION";
|
||||
|
||||
/// Probe whether this host can manipulate the OUTPUT chain at all.
|
||||
///
|
||||
/// Returns `Ok(Some(reason))` when `iptables` is missing or lacks
|
||||
/// `CAP_NET_ADMIN` (the nf_tables backend reports "Permission denied"
|
||||
/// even under `sudo` inside an unprivileged container) and the lane did
|
||||
/// not demand fault injection; returns an error when the lane demands
|
||||
/// it; returns `Ok(None)` when the blackhole can be installed.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn unavailable_reason() -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
|
||||
let id = Command::new("id").arg("-u").output()?;
|
||||
if !id.status.success() {
|
||||
return Err(format!("failed to determine the test process uid: {}", String::from_utf8_lossy(&id.stderr)).into());
|
||||
}
|
||||
let use_sudo = String::from_utf8_lossy(&id.stdout).trim() != "0";
|
||||
let mut command = if use_sudo {
|
||||
let mut command = Command::new("sudo");
|
||||
command.args(["-n", "iptables"]);
|
||||
command
|
||||
} else {
|
||||
Command::new("iptables")
|
||||
};
|
||||
let probe = command.args(["-w", "5", "-S", "OUTPUT"]).output();
|
||||
let reason = match probe {
|
||||
Ok(output) if output.status.success() => return Ok(None),
|
||||
Ok(output) => format!(
|
||||
"iptables cannot read the OUTPUT chain (status {}): {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
),
|
||||
Err(err) => format!("iptables is not runnable: {err}"),
|
||||
};
|
||||
if std::env::var_os(Self::REQUIRE_ENV).is_some() {
|
||||
return Err(format!("{} is set but network fault injection is unavailable: {reason}", Self::REQUIRE_ENV).into());
|
||||
}
|
||||
Ok(Some(reason))
|
||||
}
|
||||
|
||||
fn install(address: &str) -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
let address = address.parse::<SocketAddr>()?;
|
||||
if !address.ip().is_loopback() {
|
||||
@@ -199,6 +244,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a disk directory underneath a running server. Background writers
|
||||
/// (scanner, usage cache, heal markers) can recreate entries between the
|
||||
/// recursive listing and the final `rmdir`, which surfaces as
|
||||
/// `DirectoryNotEmpty` on macOS; retry briefly so the wipe reflects the
|
||||
/// operator action rather than a listing race.
|
||||
fn wipe_directory_while_server_runs(disk: &Path) -> std::io::Result<()> {
|
||||
let mut last_err = None;
|
||||
for _ in 0..20 {
|
||||
match std::fs::remove_dir_all(disk) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::DirectoryNotEmpty => {
|
||||
last_err = Some(err);
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(last_err.expect("retry loop only exits without success after recording an error"))
|
||||
}
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(path) else {
|
||||
return false;
|
||||
@@ -481,7 +547,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
std::fs::remove_dir_all(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
wipe_directory_while_server_runs(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
std::fs::create_dir_all(&disk0).expect("disk0 should be recreated empty while server is running");
|
||||
assert!(!has_file_under(&disk0), "disk0 must be empty immediately after runtime wipe");
|
||||
|
||||
@@ -868,6 +934,18 @@ mod tests {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_after_target_endpoint_blackhole() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
if let Some(reason) = TcpPortBlackhole::unavailable_reason()? {
|
||||
init_logging();
|
||||
warn!(
|
||||
event = "heal_interruption_skipped",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind = "target_endpoint_blackhole",
|
||||
reason,
|
||||
"Skipping endpoint blackhole scenario: network fault injection is unavailable on this host"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::TargetEndpointBlackhole),
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use super::common::{BoxError, OdmSourceSpec, OdmTestEnv, SeedObject};
|
||||
use crate::fake_s3_target::{BucketMode, Operation};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, ObjectAttributes, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -103,11 +103,19 @@ async fn get_miss_pulls_inline_and_serves_locally_afterwards() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_large_object_streams_through_and_backfills_in_background() -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let bucket = "odm-get-large";
|
||||
let env = configured_env(bucket, |spec| spec.policy.inline_max_bytes = 4096).await?;
|
||||
let env = configured_env(bucket, |spec| {
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = u64::try_from(PART_SIZE).expect("part size fits in u64");
|
||||
})
|
||||
.await?;
|
||||
let key = "large/archive.bin";
|
||||
let body = payload(512 * 1024);
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())]);
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let etag = env
|
||||
.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())])
|
||||
.remove(0);
|
||||
assert_eq!(etag.len(), 32, "the source fixture has a plain MD5 ETag");
|
||||
|
||||
let response = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(response.status, 200, "{}", String::from_utf8_lossy(&response.body));
|
||||
@@ -125,6 +133,68 @@ async fn get_large_object_streams_through_and_backfills_in_background() -> TestR
|
||||
vec![None, None],
|
||||
"one passthrough GET plus one background pull, both unranged"
|
||||
);
|
||||
|
||||
let source_requests = env.source.requests().len();
|
||||
let second_part = env.client.get_object().bucket(bucket).key(key).part_number(2).send().await?;
|
||||
assert_eq!(second_part.content_length(), Some(4096), "the completed second part is the tail");
|
||||
assert_eq!(
|
||||
second_part.content_range(),
|
||||
Some(format!("bytes {PART_SIZE}-{}/{}", body.len() - 1, body.len()).as_str()),
|
||||
"partNumber reads the stored multipart boundary"
|
||||
);
|
||||
assert_eq!(
|
||||
second_part.body.collect().await?.into_bytes(),
|
||||
body.slice(PART_SIZE..),
|
||||
"the local second part contains the exact source tail"
|
||||
);
|
||||
let third_part = env
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.part_number(3)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the completed object has exactly two parts");
|
||||
assert_eq!(third_part.code(), Some("InvalidPart"));
|
||||
|
||||
let mut part_marker = None;
|
||||
for (part_number, part_size) in [(1, PART_SIZE), (2, 4096)] {
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.max_parts(1)
|
||||
.set_part_number_marker(part_marker.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|value| value.trim_matches('"')),
|
||||
Some(etag.as_str()),
|
||||
"multipart write-back preserves the source MD5 ETag"
|
||||
);
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.expect("RustFS must expose the stored multipart layout");
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(parts.max_parts(), Some(1));
|
||||
assert_eq!(parts.is_truncated(), Some(part_number == 1));
|
||||
assert_eq!(parts.parts().len(), 1, "RustFS returns one stored part per requested page");
|
||||
assert_eq!(parts.parts()[0].part_number(), Some(part_number));
|
||||
assert_eq!(parts.parts()[0].size(), Some(i64::try_from(part_size).expect("part size fits in i64")));
|
||||
part_marker = parts.next_part_number_marker().map(str::to_owned);
|
||||
if part_number == 1 {
|
||||
assert_eq!(part_marker.as_deref(), Some("1"), "the next request continues after the first part");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
env.source.requests().len(),
|
||||
source_requests,
|
||||
"local part reads must not consult the source"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -23,16 +23,20 @@
|
||||
|
||||
use super::common::{
|
||||
ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with, start_source_rustfs,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
use crate::object_lock::common::put_object_lock_configuration;
|
||||
use crate::replication_extension_test::{
|
||||
ReplicationTargetOptions, enable_bucket_versioning, set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
ObjectAttributes, ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use local_ip_address::local_ip;
|
||||
@@ -580,6 +584,130 @@ async fn test_odm_pulled_object_replicates_and_target_as_source_is_rejected() ->
|
||||
"a bucket may not migrate from its own replication target: {}",
|
||||
rejected.body
|
||||
);
|
||||
Box::pin(assert_odm_multipart_replicates_to_rustfs(&env, bucket)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_odm_multipart_replicates_to_rustfs(env: &OdmTestEnv, bucket: &str) -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let replica = start_source_rustfs().await?;
|
||||
let replica_bucket = "odm-real-replica";
|
||||
replica.create_test_bucket(replica_bucket).await?;
|
||||
enable_bucket_versioning(&replica, replica_bucket).await?;
|
||||
let arn = set_replication_target_with_options(
|
||||
&env.rustfs,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &replica.address,
|
||||
access_key: &replica.access_key,
|
||||
secret_key: &replica.secret_key,
|
||||
target_bucket: replica_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&env.rustfs, bucket, &arn).await?;
|
||||
let mut spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
let key = "replicated/preserved-md5-multipart.bin";
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(etag.len(), 32, "the source must retain a single-PUT MD5 ETag");
|
||||
assert!(etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(env.wait_local_listed(bucket, key, SETTLE).await?, "the multipart pull must persist");
|
||||
|
||||
let deadline = Instant::now() + SETTLE;
|
||||
let source_head = loop {
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await?;
|
||||
match head.replication_status().map(|status| status.as_str()) {
|
||||
Some("COMPLETED") => break head,
|
||||
Some("FAILED") => return Err("the ODM multipart copy failed replication to RustFS".into()),
|
||||
_ => {
|
||||
assert!(Instant::now() < deadline, "the ODM multipart copy never completed replication to RustFS");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
let version = source_head
|
||||
.version_id()
|
||||
.ok_or("the versioned ODM copy omitted its version id")?;
|
||||
assert_ne!(version, "null");
|
||||
let replica_client = replica.create_s3_client();
|
||||
for (client, object_bucket) in [(&env.client, bucket), (&replica_client, replica_bucket)] {
|
||||
let attributes = client
|
||||
.get_object_attributes()
|
||||
.bucket(object_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|value| value.trim_matches('"')), Some(etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the local copy and replica must both expose two parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
}
|
||||
// REPLICA status surfaces on HEAD, like the other inbound-replica checks.
|
||||
let replica_head = replica_client
|
||||
.head_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_get = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_get.version_id(), Some(version));
|
||||
assert_eq!(replica_get.body.collect().await?.into_bytes(), body);
|
||||
let boundary = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.range(format!("bytes={}-{}", PART_SIZE - 32, PART_SIZE + 31))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(boundary.body.collect().await?.into_bytes(), body.slice(PART_SIZE - 32..PART_SIZE + 32));
|
||||
assert_eq!(
|
||||
env.source.count_requests(Operation::GetObject, key),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication and local reads must not fetch the migration source again"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -368,23 +368,23 @@ impl Drop for SlowReplicationTargetGuard {
|
||||
// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
|
||||
// shape `mc replicate resync status` decodes.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ReplicationResetStatusResponse {
|
||||
pub(crate) struct ReplicationResetStatusResponse {
|
||||
#[serde(rename = "target", default)]
|
||||
targets: Vec<ReplicationResetStatusTarget>,
|
||||
pub(crate) targets: Vec<ReplicationResetStatusTarget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ReplicationResetStatusTarget {
|
||||
pub(crate) struct ReplicationResetStatusTarget {
|
||||
#[serde(rename = "arn", default)]
|
||||
arn: String,
|
||||
pub(crate) arn: String,
|
||||
#[serde(rename = "resetid", default)]
|
||||
reset_id: String,
|
||||
pub(crate) reset_id: String,
|
||||
#[serde(rename = "resyncStatus", default)]
|
||||
status: String,
|
||||
pub(crate) status: String,
|
||||
#[serde(rename = "replicationCount", default)]
|
||||
replicated_count: i64,
|
||||
pub(crate) replicated_count: i64,
|
||||
#[serde(rename = "object", default)]
|
||||
object: String,
|
||||
pub(crate) object: String,
|
||||
}
|
||||
|
||||
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
@@ -2294,7 +2294,7 @@ async fn site_replication_state_edit(
|
||||
/// return the target `(arn, reset_id)`, asserting the response carries the
|
||||
/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
|
||||
/// that `mc replicate resync start` decodes.
|
||||
async fn start_bucket_replication_reset(
|
||||
pub(crate) async fn start_bucket_replication_reset(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
@@ -2314,7 +2314,7 @@ async fn start_bucket_replication_reset(
|
||||
Ok((arn, reset_id))
|
||||
}
|
||||
|
||||
async fn get_replication_reset_status(
|
||||
pub(crate) async fn get_replication_reset_status(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
arn: &str,
|
||||
@@ -3837,6 +3837,244 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a directory
|
||||
/// marker (`prefix/` with a body) in a versioned bucket is stored as the null
|
||||
/// version, like MinIO (`putOpts`: "for directory objects skip creating new
|
||||
/// versions"), and must still replicate to completion instead of staying
|
||||
/// `PENDING`.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_replicates_directory_marker_in_versioned_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-dir-marker-src";
|
||||
let target_bucket = "replication-dir-marker-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let marker_key = "dir/trailing/";
|
||||
let body = b"directory marker body";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(marker_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
put.version_id()
|
||||
.is_none_or(|id| id == "null" || id == uuid::Uuid::nil().to_string()),
|
||||
"a directory marker is the null version even in a versioned bucket: {:?}",
|
||||
put.version_id()
|
||||
);
|
||||
|
||||
wait_for_source_replication_status(&source_client, source_bucket, marker_key, "COMPLETED", false).await?;
|
||||
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body);
|
||||
let listed = target_client
|
||||
.list_object_versions()
|
||||
.bucket(target_bucket)
|
||||
.prefix(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
let marker_versions: Vec<_> = listed.versions().iter().filter(|v| v.key() == Some(marker_key)).collect();
|
||||
assert_eq!(marker_versions.len(), 1, "the marker must land exactly once: {marker_versions:?}");
|
||||
assert_eq!(
|
||||
marker_versions[0].version_id(),
|
||||
Some("null"),
|
||||
"the replica keeps the null version identity"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): permanently
|
||||
/// deleting a version whose payload lives in a data dir must leave the source
|
||||
/// clean once the purge replicates. Managed-SSE objects are never inlined and a
|
||||
/// plain object above the inline threshold takes the same layout. The version
|
||||
/// retained with a pending purge used to lose its data dir, so the purge state
|
||||
/// could never be applied (`VersionNotFound` on every retry) and the bucket
|
||||
/// stayed `BucketNotEmpty` while `ListObjectVersions` was already empty.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_version_purge_of_non_inline_object_releases_source_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("purge-datadir", true, true).await?;
|
||||
let target_arn = wait_for_remote_target_arn(&source_env, &source_bucket).await?;
|
||||
put_bucket_replication_with_delete_statuses(&source_env, &source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
let sse_key = "sse-object.bin";
|
||||
let large_key = "large-object.bin";
|
||||
let sse_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(sse_key)
|
||||
.body(ByteStream::from_static(b"encrypted source payload"))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let large_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(large_key)
|
||||
.body(ByteStream::from(vec![0x5a; 2 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let purged = [
|
||||
(sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
|
||||
(large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
|
||||
];
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
|
||||
for (key, version_id) in &purged {
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(*key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
// The purge state is applied on the source asynchronously after the target
|
||||
// acknowledges the delete; only then does the retained version go away and
|
||||
// the bucket become deletable. A listing that is empty while DeleteBucket
|
||||
// keeps answering BucketNotEmpty is exactly the regression.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let listing = source_client.list_object_versions().bucket(&source_bucket).send().await?;
|
||||
let listed = listing.versions().len() + listing.delete_markers().len();
|
||||
match source_client.delete_bucket().bucket(&source_bucket).send().await {
|
||||
Ok(_) => break,
|
||||
Err(err) if err.code() == Some("BucketNotEmpty") => {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"source bucket stayed BucketNotEmpty after the version purge replicated; \
|
||||
ListObjectVersions shows {listed} entries"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a single-part
|
||||
/// object uploaded with `x-amz-checksum-*` must reach the target with the same
|
||||
/// checksum. The outbound options keyed the stored record by algorithm name,
|
||||
/// which the target client sent as `x-amz-meta-*` user metadata, so a replica
|
||||
/// never carried a checksum although the source HEAD returned one.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_forwards_single_part_object_checksums() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-checksum-src";
|
||||
let target_bucket = "replication-checksum-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let body = b"123456789";
|
||||
let crc32_key = "checksum-crc32.txt";
|
||||
let sha256_key = "checksum-sha256.txt";
|
||||
let crc32_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(crc32_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_crc32 = crc32_put.checksum_crc32().ok_or("source PUT omitted CRC32")?.to_string();
|
||||
let sha256_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(sha256_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_sha256 = sha256_put.checksum_sha256().ok_or("source PUT omitted SHA256")?.to_string();
|
||||
|
||||
for key in [crc32_key, sha256_key] {
|
||||
wait_for_source_replication_status(&source_client, source_bucket, key, "COMPLETED", false).await?;
|
||||
}
|
||||
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(crc32_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.checksum_crc32(), Some(expected_crc32.as_str()), "replica lost the CRC32 checksum");
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(sha256_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica.checksum_sha256(),
|
||||
Some(expected_sha256.as_str()),
|
||||
"replica lost the SHA256 checksum"
|
||||
);
|
||||
// The bare algorithm name must not leak as user metadata either.
|
||||
assert!(
|
||||
replica
|
||||
.metadata()
|
||||
.is_none_or(|meta| !meta.keys().any(|k| k.eq_ignore_ascii_case("sha256"))),
|
||||
"replica carries the checksum as user metadata: {:?}",
|
||||
replica.metadata()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -31,17 +31,20 @@
|
||||
//! Adding a target behavior the fleet has shown: add the mode to the fake
|
||||
//! target, add a row here, and record any cell that is red before the fix.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::fake_source_client;
|
||||
use crate::common::{init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::{OdmEnvOptions, OdmTestEnv, fake_source_client};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, put_bucket_replication,
|
||||
set_replication_target_with_options,
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, get_replication_reset_status,
|
||||
put_bucket_replication, set_replication_target_with_options, start_bucket_replication_reset,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockMode};
|
||||
use aws_sdk_s3::types::{
|
||||
Checksum, ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ObjectAttributes, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -110,16 +113,23 @@ enum ObjectShape {
|
||||
/// Two-part multipart upload with a GOVERNANCE retention period; the
|
||||
/// lock headers travel on CreateMultipartUpload, which has no body.
|
||||
LockedMultipart,
|
||||
/// ODM stores two local parts while preserving a single-PUT source's MD5 ETag.
|
||||
OdmPreservedMd5Multipart,
|
||||
/// Single-part object uploaded with `x-amz-checksum-sha256`; the replica
|
||||
/// must carry the same header (rustfs/backlog#2340).
|
||||
Checksummed,
|
||||
}
|
||||
|
||||
impl ObjectShape {
|
||||
const ALL: [ObjectShape; 6] = [
|
||||
const ALL: [ObjectShape; 8] = [
|
||||
ObjectShape::Empty,
|
||||
ObjectShape::Plain,
|
||||
ObjectShape::Retention,
|
||||
ObjectShape::LegalHold,
|
||||
ObjectShape::Multipart,
|
||||
ObjectShape::LockedMultipart,
|
||||
ObjectShape::OdmPreservedMd5Multipart,
|
||||
ObjectShape::Checksummed,
|
||||
];
|
||||
|
||||
fn key(self) -> &'static str {
|
||||
@@ -130,6 +140,17 @@ impl ObjectShape {
|
||||
ObjectShape::LegalHold => "matrix/legal-hold.bin",
|
||||
ObjectShape::Multipart => "matrix/multipart.bin",
|
||||
ObjectShape::LockedMultipart => "matrix/locked-multipart.bin",
|
||||
ObjectShape::OdmPreservedMd5Multipart => "matrix/odm-preserved-md5.bin",
|
||||
ObjectShape::Checksummed => "matrix/checksummed.bin",
|
||||
}
|
||||
}
|
||||
|
||||
/// The `x-amz-checksum-*` header the source stored and every upload of
|
||||
/// the replica must repeat.
|
||||
fn forwarded_checksum_header(self) -> Option<&'static str> {
|
||||
match self {
|
||||
ObjectShape::Checksummed => Some("x-amz-checksum-sha256"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +160,8 @@ impl ObjectShape {
|
||||
|
||||
/// Upload the shape to the source and return the bytes the target must
|
||||
/// end up holding.
|
||||
async fn put(self, client: &Client, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
async fn put(self, env: &OdmTestEnv, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
let client = &env.client;
|
||||
let key = self.key();
|
||||
match self {
|
||||
ObjectShape::Empty => {
|
||||
@@ -190,6 +212,19 @@ impl ObjectShape {
|
||||
}
|
||||
ObjectShape::Multipart => multipart_put(client, bucket, key, 0x44, false).await,
|
||||
ObjectShape::LockedMultipart => multipart_put(client, bucket, key, 0x55, true).await,
|
||||
ObjectShape::OdmPreservedMd5Multipart => odm_preserved_md5_multipart(env, bucket, key).await,
|
||||
ObjectShape::Checksummed => {
|
||||
let body = payload(40 * 1024, 0x66);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,6 +254,170 @@ fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation {
|
||||
.unwrap_or(Expectation::Completed)
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340: a target that mints its own version ids (Wasabi,
|
||||
/// AWS S3) answers 404 to a HEAD by the source uuid, which the worker used to
|
||||
/// read as "replica missing" and re-drive the PUT — one more target version
|
||||
/// per heal, MRF retry or resync. Two re-drive shapes, both must converge on
|
||||
/// the single version the first PUT created:
|
||||
/// - the first PUT lands but its response is lost, so the object is FAILED
|
||||
/// and the scanner heal pass re-drives it;
|
||||
/// - an existing-object resync re-drives a COMPLETED object unconditionally.
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-redrive-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what re-drives a FAILED object.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-mint-own-redrive-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
// Teach the worker the target's identity contract with one ordinary
|
||||
// write, exactly as production learns it (the PUT response carries the
|
||||
// minted id).
|
||||
let probe_key = "redrive/identity-probe.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(probe_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, probe_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
|
||||
// Shape 1: the PUT is stored, its response never arrives, heal re-drives.
|
||||
let heal_key = "redrive/heal.bin";
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, heal_key, FakeTargetFault::DisconnectAfterResponse, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(heal_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x02)))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replication_status_and_single_version(&source_client, source_bucket, &target, &target_bucket, heal_key).await?;
|
||||
|
||||
// Shape 2: an existing-object resync re-drives a COMPLETED object.
|
||||
let resync_key = "redrive/resync.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(resync_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x03)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, resync_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let (reset_arn, _reset_id) = start_bucket_replication_reset(source_env, source_bucket).await?;
|
||||
assert_eq!(reset_arn, target_arn);
|
||||
let resync = async {
|
||||
loop {
|
||||
let status = get_replication_reset_status(source_env, source_bucket, &target_arn).await?;
|
||||
if let Some(entry) = status.targets.iter().find(|entry| entry.arn == target_arn)
|
||||
&& entry.status == "Completed"
|
||||
{
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(entry.replicated_count);
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
let replicated = timeout(Duration::from_secs(90), resync)
|
||||
.await
|
||||
.map_err(|_| "existing-object resync did not complete within 90 seconds")??;
|
||||
assert!(replicated >= 3, "resync must count the located replicas as replicated, got {replicated}");
|
||||
for key in [probe_key, heal_key, resync_key] {
|
||||
let versions = target.stored_versions(&target_bucket, key);
|
||||
assert_eq!(
|
||||
versions.len(),
|
||||
1,
|
||||
"{key}: a re-drive against a target that mints its own version ids must not mint another one: {versions:?}"
|
||||
);
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait until `key` is COMPLETED on the source and, for the observation
|
||||
/// window after that, the target still holds exactly one live version of it.
|
||||
async fn wait_for_replication_status_and_single_version(
|
||||
source_client: &Client,
|
||||
source_bucket: &str,
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
key: &str,
|
||||
) -> TestResult {
|
||||
// The lost PUT response first settles the object FAILED; only the next
|
||||
// scanner heal pass can turn that into COMPLETED, so FAILED is transient
|
||||
// here and the wait is for COMPLETED alone.
|
||||
let converged = async {
|
||||
loop {
|
||||
let head = source_client.head_object().bucket(source_bucket).key(key).send().await?;
|
||||
if head.replication_status().is_some_and(|status| status.as_str() == "COMPLETED") {
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
timeout(Duration::from_secs(90), converged)
|
||||
.await
|
||||
.map_err(|_| format!("{key}: heal re-drive did not converge to COMPLETED within 90 seconds"))??;
|
||||
// The heal pass keeps visiting the key for a few scanner cycles; a
|
||||
// duplicate would show up here as a second stored version.
|
||||
for _ in 0..12 {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
assert_eq!(versions.len(), 1, "{key}: target minted another version on re-drive: {versions:?}");
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn matrix_baseline_target() -> TestResult {
|
||||
run_row(TargetMode::Baseline).await
|
||||
@@ -270,11 +469,15 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
mode.apply(&target);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = format!("matrix-{}-src", mode.slug());
|
||||
let source_client = source_env.create_s3_client();
|
||||
@@ -284,9 +487,9 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(&source_env, &source_bucket).await?;
|
||||
enable_bucket_versioning(source_env, &source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_env,
|
||||
&source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
@@ -299,14 +502,21 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, &source_bucket, &target_arn).await?;
|
||||
put_bucket_replication(source_env, &source_bucket, &target_arn).await?;
|
||||
|
||||
let target_client = fake_source_client(&target);
|
||||
let mut failures = Vec::new();
|
||||
for shape in ObjectShape::ALL {
|
||||
let cell = format!("{}/{:?}", mode.slug(), shape);
|
||||
let expected_body = shape.put(&source_client, &source_bucket).await?;
|
||||
let expected_body = shape.put(&env, &source_bucket).await?;
|
||||
let status = wait_for_terminal_replication_status(&source_client, &source_bucket, shape.key()).await?;
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
assert_eq!(
|
||||
env.source.count_requests(FakeTargetOperation::GetObject, shape.key()),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication must read the persisted local parts"
|
||||
);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let outcome = match expectation(mode, shape) {
|
||||
Expectation::Completed => {
|
||||
@@ -379,6 +589,36 @@ async fn check_completed_cell(
|
||||
if uploads.is_empty() {
|
||||
return Err("no upload reached the target although the source reports COMPLETED".into());
|
||||
}
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(shape.key()))
|
||||
.collect();
|
||||
for operation in [
|
||||
FakeTargetOperation::CreateMultipartUpload,
|
||||
FakeTargetOperation::CompleteMultipartUpload,
|
||||
] {
|
||||
if !key_requests.iter().any(|record| record.operation == operation) {
|
||||
return Err(format!("preserved-MD5 multipart object did not use {operation:?}").into());
|
||||
}
|
||||
}
|
||||
if key_requests
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
{
|
||||
return Err("preserved-MD5 multipart object used a single PutObject".into());
|
||||
}
|
||||
let mut part_numbers: Vec<_> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.map(|record| record.part_number)
|
||||
.collect();
|
||||
part_numbers.sort_unstable();
|
||||
part_numbers.dedup();
|
||||
if part_numbers != [Some(1), Some(2)] {
|
||||
return Err(format!("preserved-MD5 multipart object uploaded unexpected parts: {part_numbers:?}").into());
|
||||
}
|
||||
}
|
||||
if let Some(framed) = uploads.iter().find(|record| record.transport.aws_chunked) {
|
||||
return Err(format!("{cell}: an upload went out aws-chunked (rustfs#6853 framing): {framed:?}").into());
|
||||
}
|
||||
@@ -401,6 +641,19 @@ async fn check_completed_cell(
|
||||
}) {
|
||||
return Err(format!("a locked PutObject went out without any integrity header (rustfs#7082): {bare:?}").into());
|
||||
}
|
||||
// rustfs/backlog#2340 contract: a source checksum reaches the target as
|
||||
// the `x-amz-checksum-*` header, not as user metadata; every PutObject of
|
||||
// the shape carries it.
|
||||
if let Some(header) = shape.forwarded_checksum_header()
|
||||
&& let Some(missing) = uploads.iter().find(|record| {
|
||||
record.operation == FakeTargetOperation::PutObject
|
||||
&& !record.transport.checksum_headers.iter().any(|name| name == header)
|
||||
})
|
||||
{
|
||||
return Err(
|
||||
format!("a PutObject went out without the source's {header} header (rustfs/backlog#2340): {missing:?}").into(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -455,6 +708,71 @@ async fn wait_for_terminal_replication_status(
|
||||
}
|
||||
}
|
||||
|
||||
async fn odm_preserved_md5_multipart(env: &OdmTestEnv, bucket: &str, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let origin_bucket = format!("{bucket}-origin");
|
||||
env.source.create_bucket_with_mode(&origin_bucket, BucketMode::Unversioned);
|
||||
let mut spec = env.fake_source_spec(&origin_bucket);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
// A normal source PUT produces the MD5 ETag; only ODM chooses the local parts.
|
||||
let body = payload(PART_SIZE + 4096, 0x66);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(&origin_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let source_etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(source_etag.len(), 32, "source fixture must have a single-PUT MD5 ETag");
|
||||
assert!(source_etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(
|
||||
env.wait_local_listed(bucket, key, Duration::from_secs(30)).await?,
|
||||
"ODM must persist the object"
|
||||
);
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Checksum)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|etag| etag.trim_matches('"')), Some(source_etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the ODM copy must expose its two local parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
assert!(
|
||||
attributes
|
||||
.checksum()
|
||||
.is_none_or(|checksum| checksum == &Checksum::builder().build()),
|
||||
"multipart routing must work without an object checksum record"
|
||||
);
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
async fn multipart_put(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
|
||||
@@ -14,9 +14,10 @@
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
|
||||
signed_request,
|
||||
};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject, fake_source_client};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
|
||||
};
|
||||
@@ -25,9 +26,10 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectAttributes, ObjectLockConfiguration,
|
||||
ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -1205,3 +1207,724 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// rc.5 multipart layouts under the current build (backlog#2147 follow-up to
|
||||
// rustfs#7305)
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// rustfs#7305 changed `ObjectInfo::is_multipart` to consult the stored part
|
||||
// list before the ETag shape. Every earlier check of that change used
|
||||
// synthetic metadata; this scenario writes the layouts with the published
|
||||
// rc.5 binary and then reads, describes, and replicates them with the
|
||||
// current build on the same data directory.
|
||||
|
||||
const LAYOUT_PLAIN_BUCKET: &str = "upgrade-layout-plain";
|
||||
const LAYOUT_ENCRYPTED_BUCKET: &str = "upgrade-layout-encrypted";
|
||||
const LAYOUT_REPLICA_BUCKET: &str = "upgrade-layout-replica";
|
||||
const LAYOUT_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LAYOUT_TAIL_SIZE: usize = 1024 * 1024 + 4096;
|
||||
const LAYOUT_SSEC_KEY: &str = "0123456789abcdef0123456789abcdef";
|
||||
const LAYOUT_REPLICATION_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
|
||||
struct LayoutCase {
|
||||
bucket: &'static str,
|
||||
key: &'static str,
|
||||
/// Empty for a single PUT.
|
||||
part_sizes: Vec<usize>,
|
||||
body: Vec<u8>,
|
||||
ssec: bool,
|
||||
/// `false` for layouts whose replication is a known pre-existing failure;
|
||||
/// their outcome is logged, not asserted.
|
||||
assert_replication: bool,
|
||||
/// Recorded from the rc.5 writer.
|
||||
rc5_etag: String,
|
||||
/// Whether rc.5 reported `ObjectParts` for the object.
|
||||
rc5_reported_parts: Option<usize>,
|
||||
}
|
||||
|
||||
impl LayoutCase {
|
||||
fn is_multipart_layout(&self) -> bool {
|
||||
self.part_sizes.len() > 1
|
||||
}
|
||||
|
||||
fn label(&self) -> String {
|
||||
format!("{}/{}", self.bucket, self.key)
|
||||
}
|
||||
}
|
||||
|
||||
fn layout_noise(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut state = seed ^ 0x9E37_79B9_7F4A_7C15;
|
||||
(0..len)
|
||||
.map(|_| {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
(state >> 24) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn layout_text(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len + 64);
|
||||
let mut line = 0u64;
|
||||
while out.len() < len {
|
||||
out.extend_from_slice(format!("rc5 legacy layout seed={seed} line={line} lorem ipsum dolor sit amet\n").as_bytes());
|
||||
line += 1;
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
}
|
||||
|
||||
fn layout_ssec_key_md5() -> String {
|
||||
use md5::{Digest as _, Md5};
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(LAYOUT_SSEC_KEY.as_bytes());
|
||||
base64_simd::STANDARD.encode_to_string(hasher.finalize())
|
||||
}
|
||||
|
||||
fn layout_ssec_key() -> String {
|
||||
base64_simd::STANDARD.encode_to_string(LAYOUT_SSEC_KEY)
|
||||
}
|
||||
|
||||
async fn layout_head(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::head_object::HeadObjectOutput, BoxError> {
|
||||
let request = client.head_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_get(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
) -> Result<aws_sdk_s3::operation::get_object::GetObjectOutput, BoxError> {
|
||||
let request = client.get_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = request.set_range(range).set_part_number(part_number);
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_attributes(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput, BoxError> {
|
||||
let request = client
|
||||
.get_object_attributes()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::ObjectSize)
|
||||
.max_parts(100);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
/// Write `case` with the rc.5 client; single PUT when `part_sizes` is empty.
|
||||
async fn layout_write(client: &Client, case: &LayoutCase) -> Result<(), BoxError> {
|
||||
let content_type = "text/plain";
|
||||
if case.part_sizes.is_empty() {
|
||||
let request = client
|
||||
.put_object()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type)
|
||||
.body(ByteStream::from(case.body.clone()));
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
request.send().await?;
|
||||
return Ok(());
|
||||
}
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type);
|
||||
let create = if case.ssec {
|
||||
create
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
create
|
||||
};
|
||||
let created = create.send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
let mut completed = Vec::with_capacity(case.part_sizes.len());
|
||||
let mut offset = 0usize;
|
||||
for (index, size) in case.part_sizes.iter().enumerate() {
|
||||
let part_number = i32::try_from(index + 1)?;
|
||||
let chunk = case.body[offset..offset + size].to_vec();
|
||||
offset += size;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(chunk));
|
||||
let upload = if case.ssec {
|
||||
upload
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
upload
|
||||
};
|
||||
let uploaded = upload.send().await?;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().ok_or("UploadPart omitted ETag")?)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn layout_cases() -> Vec<LayoutCase> {
|
||||
let two = vec![LAYOUT_PART_SIZE, LAYOUT_TAIL_SIZE];
|
||||
let three = vec![LAYOUT_PART_SIZE, LAYOUT_PART_SIZE, 4096];
|
||||
let total = |sizes: &[usize]| sizes.iter().sum::<usize>();
|
||||
let case = |bucket, key, part_sizes: Vec<usize>, body: Vec<u8>, ssec| LayoutCase {
|
||||
bucket,
|
||||
key,
|
||||
part_sizes,
|
||||
body,
|
||||
ssec,
|
||||
assert_replication: true,
|
||||
rc5_etag: String::new(),
|
||||
rc5_reported_parts: None,
|
||||
};
|
||||
vec![
|
||||
case(LAYOUT_PLAIN_BUCKET, "plain/single.bin", vec![], layout_noise(1024 * 1024 + 17, 1), false),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 2),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 3),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-single.txt",
|
||||
vec![],
|
||||
layout_text(1024 * 1024 + 17, 4),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 5),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 6),
|
||||
true,
|
||||
),
|
||||
// SSE-C passthrough replicates the stored ciphertext part by part; a
|
||||
// compressible first part is stored well below 5 MiB and a standard
|
||||
// target rejects it with EntityTooSmall. rc.5 fails the same way (see
|
||||
// `rc5_baseline_replicates_multipart_layouts`), so the outcome is
|
||||
// recorded rather than asserted here; tracked as rustfs/backlog#2363.
|
||||
LayoutCase {
|
||||
assert_replication: false,
|
||||
..case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 7),
|
||||
true,
|
||||
)
|
||||
},
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/single.bin",
|
||||
vec![],
|
||||
layout_noise(1024 * 1024 + 17, 8),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 9),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 10),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 11),
|
||||
false,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn layout_server_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = bucket_config_server_env();
|
||||
env.push(("RUSTFS_COMPRESSION_ENABLED", "true"));
|
||||
env.push(("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"));
|
||||
env
|
||||
}
|
||||
|
||||
fn layout_reported_parts(attributes: &aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput) -> Option<usize> {
|
||||
attributes.object_parts().map(|parts| parts.parts().len())
|
||||
}
|
||||
|
||||
async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let head = layout_head(client, case).await?;
|
||||
assert_eq!(
|
||||
head.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: the ETag written by rc.5 must be reported unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
head.content_length(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: HEAD content length"
|
||||
);
|
||||
|
||||
let full = layout_get(client, case, None, None).await?.body.collect().await?.into_bytes();
|
||||
assert_eq!(full.len(), case.body.len(), "{context}: {label}: full GET length");
|
||||
assert!(full == case.body, "{context}: {label}: full GET body must equal the rc.5 upload");
|
||||
|
||||
if case.is_multipart_layout() {
|
||||
let first = case.part_sizes[0];
|
||||
let range = format!("bytes={}-{}", first - 32, first + 31);
|
||||
let crossing = layout_get(client, case, Some(range), None)
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert!(
|
||||
crossing == case.body[first - 32..first + 32],
|
||||
"{context}: {label}: range across the first part boundary"
|
||||
);
|
||||
let tail_start: usize = case.part_sizes[..case.part_sizes.len() - 1].iter().sum();
|
||||
let last_number = i32::try_from(case.part_sizes.len())?;
|
||||
let last = layout_get(client, case, None, Some(last_number)).await?;
|
||||
assert_eq!(
|
||||
last.content_length(),
|
||||
Some(i64::try_from(case.part_sizes[case.part_sizes.len() - 1])?),
|
||||
"{context}: {label}: partNumber={last_number} length"
|
||||
);
|
||||
let last_body = last.body.collect().await?.into_bytes();
|
||||
assert!(
|
||||
last_body == case.body[tail_start..],
|
||||
"{context}: {label}: partNumber={last_number} body must be the stored last part"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_layout_attributes(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let attributes = layout_attributes(client, case).await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: attributes ETag"
|
||||
);
|
||||
assert_eq!(
|
||||
attributes.object_size(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: attributes ObjectSize"
|
||||
);
|
||||
if case.is_multipart_layout() {
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or_else(|| format!("{context}: {label}: multipart layout must expose ObjectParts"))?;
|
||||
assert_eq!(
|
||||
parts.total_parts_count(),
|
||||
Some(i32::try_from(case.part_sizes.len())?),
|
||||
"{context}: {label}: TotalPartsCount"
|
||||
);
|
||||
let observed: Vec<(Option<i32>, Option<i64>)> =
|
||||
parts.parts().iter().map(|part| (part.part_number(), part.size())).collect();
|
||||
let expected: Vec<(Option<i32>, Option<i64>)> = case
|
||||
.part_sizes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, size)| (Some(index as i32 + 1), Some(*size as i64)))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
observed, expected,
|
||||
"{context}: {label}: ObjectParts must report the plaintext part layout"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
attributes.object_parts().is_none_or(|parts| parts.parts().is_empty()),
|
||||
"{context}: {label}: a single PUT must not report stored parts"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_layout_replication_rule(env: &RustFSTestEnvironment, bucket: &str, arn: &str) -> TestResult {
|
||||
let body = format!(
|
||||
r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Role></Role>
|
||||
<Rule>
|
||||
<ID>legacy-layouts</ID>
|
||||
<Priority>1</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
</ReplicationConfiguration>"#
|
||||
);
|
||||
let url = format!("{}/{bucket}?replication", env.url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(body.into_bytes()),
|
||||
Some("application/xml"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("put replication rule on {bucket} failed: {status} {body}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for the existing-object replication of `case` to reach a terminal
|
||||
/// status and return it (`COMPLETED` or `FAILED`).
|
||||
async fn wait_layout_replication_terminal(client: &Client, case: &LayoutCase) -> Result<String, BoxError> {
|
||||
let deadline = Instant::now() + LAYOUT_REPLICATION_TIMEOUT;
|
||||
loop {
|
||||
let head = layout_head(client, case).await?;
|
||||
let status = head.replication_status().map(|status| status.as_str().to_string());
|
||||
if matches!(status.as_deref(), Some("COMPLETED") | Some("FAILED")) {
|
||||
return Ok(status.unwrap_or_default());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"{}: existing-object replication never reached a terminal status; last {status:?}",
|
||||
case.label()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct LayoutTransport {
|
||||
status: String,
|
||||
uploaded_parts: Vec<i32>,
|
||||
single_puts: usize,
|
||||
completes: usize,
|
||||
/// Raw per-key journal in target order: (sequence, operation, part number,
|
||||
/// upload id), so duplicate drives can be told apart from retries.
|
||||
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
|
||||
}
|
||||
|
||||
/// Configure every layout bucket to replicate its existing objects to a fresh
|
||||
/// fake target, wait for each case to settle, and report the transport the
|
||||
/// target observed per case.
|
||||
async fn replicate_layouts(
|
||||
env: &RustFSTestEnvironment,
|
||||
client: &Client,
|
||||
cases: &[LayoutCase],
|
||||
) -> Result<(FakeS3Target, Vec<LayoutTransport>), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(LAYOUT_REPLICA_BUCKET);
|
||||
for bucket in [LAYOUT_PLAIN_BUCKET, LAYOUT_ENCRYPTED_BUCKET] {
|
||||
let arn = set_replication_target_with_options(
|
||||
env,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: LAYOUT_REPLICA_BUCKET,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_layout_replication_rule(env, bucket, &arn).await?;
|
||||
}
|
||||
let mut statuses = Vec::with_capacity(cases.len());
|
||||
for case in cases {
|
||||
statuses.push(wait_layout_replication_terminal(client, case).await?);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let mut transports = Vec::with_capacity(cases.len());
|
||||
for (case, status) in cases.iter().zip(statuses) {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(case.key))
|
||||
.collect();
|
||||
let mut uploaded_parts: Vec<i32> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.filter_map(|record| record.part_number)
|
||||
.collect();
|
||||
uploaded_parts.sort_unstable();
|
||||
uploaded_parts.dedup();
|
||||
let transport = LayoutTransport {
|
||||
status,
|
||||
uploaded_parts,
|
||||
single_puts: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.count(),
|
||||
completes: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::CompleteMultipartUpload)
|
||||
.count(),
|
||||
journal: key_requests
|
||||
.iter()
|
||||
.map(|record| {
|
||||
(
|
||||
record.sequence,
|
||||
format!("{:?}", record.operation),
|
||||
record.part_number,
|
||||
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
?transport,
|
||||
"replication transport observed on the target"
|
||||
);
|
||||
transports.push(transport);
|
||||
}
|
||||
Ok((target, transports))
|
||||
}
|
||||
|
||||
/// rc.5 writes single-PUT, multipart, compressed, SSE-C and SSE-S3 layouts;
|
||||
/// the current build must read every byte, expose the stored part layout
|
||||
/// through GetObjectAttributes and partNumber reads, and replicate the objects
|
||||
/// with the transport that matches their stored parts.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn direct_upgrade_from_rc5_preserves_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let old_client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
assert_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET, "rc.5").await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&old_client, case).await?;
|
||||
let head = layout_head(&old_client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
case.rc5_reported_parts = layout_attributes(&old_client, case)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|a| layout_reported_parts(&a));
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
parts = case.part_sizes.len(),
|
||||
etag = %case.rc5_etag,
|
||||
rc5_reported_parts = ?case.rc5_reported_parts,
|
||||
"rc.5 wrote a legacy layout"
|
||||
);
|
||||
}
|
||||
// The rc.5 writer must itself still read what it wrote, so a later
|
||||
// failure is attributable to the upgrade rather than to the fixture.
|
||||
for case in &cases {
|
||||
assert_layout_readable(&old_client, case, "rc.5").await?;
|
||||
}
|
||||
|
||||
// Upgrade in place.
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
for case in &cases {
|
||||
assert_layout_readable(&client, case, "upgraded").await?;
|
||||
assert_layout_attributes(&client, case, "upgraded").await?;
|
||||
}
|
||||
|
||||
// Replicate the pre-existing objects with the current build.
|
||||
let (target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let replica_client = fake_source_client(&target);
|
||||
for (case, transport) in cases.iter().zip(&transports) {
|
||||
let label = case.label();
|
||||
if !case.assert_replication {
|
||||
continue;
|
||||
}
|
||||
assert_eq!(transport.status, "COMPLETED", "{label}: existing-object replication must complete");
|
||||
if case.is_multipart_layout() {
|
||||
let expected: Vec<i32> = (1..=i32::try_from(case.part_sizes.len())?).collect();
|
||||
assert_eq!(
|
||||
transport.uploaded_parts, expected,
|
||||
"{label}: stored parts must replicate as the same multipart layout"
|
||||
);
|
||||
// The current build can drive an existing object twice (two
|
||||
// full CreateMultipartUpload/UploadPart/Complete rounds with
|
||||
// distinct upload ids) while its status is still PENDING; the
|
||||
// rc.5 baseline drives once. That is a scheduling difference,
|
||||
// not a layout one, tracked as rustfs/backlog#2362.
|
||||
assert!(transport.completes >= 1, "{label}: at least one CompleteMultipartUpload");
|
||||
if transport.completes > 1 {
|
||||
tracing::warn!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %label,
|
||||
completes = transport.completes,
|
||||
journal = ?transport.journal,
|
||||
"existing-object replication drove the same object more than once (rustfs/backlog#2362)"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
transport.single_puts, 0,
|
||||
"{label}: a multipart layout must not go out as a single PutObject"
|
||||
);
|
||||
} else {
|
||||
assert!(transport.single_puts >= 1, "{label}: a single PUT replicates as PutObject");
|
||||
assert!(transport.uploaded_parts.is_empty(), "{label}: a single PUT must not go out as multipart");
|
||||
}
|
||||
if !case.ssec {
|
||||
let replica = replica_client
|
||||
.get_object()
|
||||
.bucket(LAYOUT_REPLICA_BUCKET)
|
||||
.key(case.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("{label}: replica missing on the target: {err}"))?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(replica.len(), case.body.len(), "{label}: replica length");
|
||||
assert!(replica == case.body, "{label}: replica body must equal the rc.5 upload");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The same layouts replicated by rc.5 itself, without an upgrade. This is the
|
||||
/// baseline that tells a pre-existing transport failure apart from one the
|
||||
/// current build introduced; it records the outcome per layout and only fails
|
||||
/// when the fixture cannot run.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn rc5_baseline_replicates_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&client, case).await?;
|
||||
let head = layout_head(&client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
}
|
||||
let (_target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let summary: Vec<String> = cases
|
||||
.iter()
|
||||
.zip(&transports)
|
||||
.map(|(case, transport)| {
|
||||
format!(
|
||||
"{}: {} parts={:?} puts={}",
|
||||
case.label(),
|
||||
transport.status,
|
||||
transport.uploaded_parts,
|
||||
transport.single_puts
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
tracing::info!(target: "e2e_test::upgrade_compatibility_test", ?summary, "rc.5 baseline replication outcomes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -118,8 +118,6 @@ hotpath-cpu = [
|
||||
# injection, xl.meta transition assertions) via `api::tier::test_util`.
|
||||
# Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6).
|
||||
test-util = []
|
||||
# Observes real startup CAS only in the dedicated E2E binary.
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
@@ -228,7 +226,7 @@ metrics = { workspace = true }
|
||||
# crates.io. The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.1"
|
||||
rustfs-uring = "0.2.2"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, VersionIdentityCapability, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -76,11 +76,26 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_disposition {
|
||||
pub use crate::bucket::lifecycle::recovery_disposition::{
|
||||
IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, IlmRecoveryDispositionState,
|
||||
dry_run_recovery_disposition, execute_recovery_disposition,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_export {
|
||||
pub use crate::bucket::lifecycle::recovery_export::{
|
||||
IlmRecoveryExportCreated, IlmRecoveryExportObservation, create_recovery_export,
|
||||
inspect_recovery_export_observation, load_recovery_export,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
|
||||
inspect_transition_transaction_for_operator, retry_transition_recovery_for_operator,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
@@ -293,7 +308,7 @@ pub mod cache {
|
||||
pub mod capacity {
|
||||
pub use crate::core::pools::{
|
||||
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
is_pool_activation_fleet_proof_error, path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
};
|
||||
pub use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
}
|
||||
@@ -368,8 +383,6 @@ pub mod data_usage {
|
||||
pub mod disk {
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
pub use crate::disk::local::ScanGuard;
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
@@ -448,9 +461,12 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, IlmRecoveryExportFleetProofToken,
|
||||
LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, acquire_ilm_recovery_export_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_local_process_epoch,
|
||||
ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
@@ -503,7 +519,8 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
|
||||
ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX,
|
||||
TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
@@ -546,8 +563,8 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
use crate::bucket::target::ARN;
|
||||
use crate::bucket::target::BucketTargetType;
|
||||
@@ -126,6 +126,22 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
}
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
|
||||
/// Whether an edited bucket target still addresses the same remote service
|
||||
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
|
||||
/// about that service stays valid across the edit.
|
||||
fn same_replication_service(edited: &BucketTarget, previous: &BucketTarget) -> bool {
|
||||
let access_key = |target: &BucketTarget| target.credentials.as_ref().map(|credentials| credentials.access_key.clone());
|
||||
edited.endpoint == previous.endpoint
|
||||
&& edited.target_bucket == previous.target_bucket
|
||||
&& edited.secure == previous.secure
|
||||
&& edited.path == previous.path
|
||||
&& access_key(edited) == access_key(previous)
|
||||
}
|
||||
|
||||
/// Page size and page budget for [`TargetClient::find_version_by_etag`].
|
||||
const FIND_VERSION_BY_ETAG_PAGE_SIZE: i32 = 1000;
|
||||
const FIND_VERSION_BY_ETAG_MAX_PAGES: usize = 8;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
|
||||
@@ -349,6 +365,13 @@ struct TargetClientBuildProbe {
|
||||
/// their import path while the verdict vocabulary lives with the
|
||||
/// replication decision logic.
|
||||
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||
/// Version-identity verdicts (see the enum's own docs in
|
||||
/// `rustfs-replication`) are cached here per target ARN and follow the same
|
||||
/// `arn_remotes_map` lifecycle. They carry no TTL: the verdict is refreshed
|
||||
/// by every replication write's response, so it can only go stale on a
|
||||
/// target that receives no writes — and a stale `MintsOwn` costs one extra
|
||||
/// content-identity lookup before a PUT, never a lost replica.
|
||||
pub use crate::bucket::replication::VersionIdentityCapability;
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
@@ -375,6 +398,11 @@ pub struct BucketTargetSys {
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
/// Version-identity verdicts keyed by target ARN. See
|
||||
/// [`VersionIdentityCapability`]; reset alongside `arn_remotes_map`. A std
|
||||
/// lock (never held across an await) so the replication worker can record
|
||||
/// a verdict from inside its synchronous PUT-response audit.
|
||||
version_identity_map: Arc<std::sync::RwLock<HashMap<String, VersionIdentityCapability>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
|
||||
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
|
||||
@@ -423,6 +451,7 @@ impl BucketTargetSys {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
version_identity_map: Arc::new(std::sync::RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -746,10 +775,40 @@ impl BucketTargetSys {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached version-identity verdict for a target ARN; `Unknown` until a
|
||||
/// replication write or a replication-check VersionFidelity probe judged
|
||||
/// it since the target was built.
|
||||
pub fn version_identity_capability(&self, arn: &str) -> VersionIdentityCapability {
|
||||
self.version_identity_map
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(arn)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Record a version-identity verdict for a target ARN. Written by the
|
||||
/// replication worker after every PutObject / CompleteMultipartUpload
|
||||
/// response and by the replication-check VersionFidelity phase.
|
||||
pub fn record_version_identity_capability(&self, arn: &str, capability: VersionIdentityCapability) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(arn.to_string(), capability);
|
||||
}
|
||||
|
||||
fn forget_version_identity_capability(&self, arn: &str) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(arn);
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
@@ -1162,12 +1221,32 @@ impl BucketTargetSys {
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
let unchanged_service: HashMap<&str, &BucketTarget> = targets
|
||||
.map(|new_targets| {
|
||||
new_targets
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| (target.arn.as_str(), target))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
// The version-identity verdict survives an edit that keeps the
|
||||
// same remote service (a resync start or a bandwidth change
|
||||
// rewrites the entry in place): forgetting it there would make
|
||||
// the very resync that follows re-drive every object as a
|
||||
// duplicate on a target that mints its own version ids.
|
||||
if unchanged_service
|
||||
.get(target.arn.as_str())
|
||||
.is_none_or(|edited| !same_replication_service(edited, &target))
|
||||
{
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1892,6 +1971,62 @@ impl TargetClient {
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// Locate a replica by content identity on a target that mints its own
|
||||
/// version ids: page `ListObjectVersions` under the exact key and return
|
||||
/// the newest live version whose ETag matches `source_etag`. Delete
|
||||
/// markers and prefix siblings never match. Bounded to
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_PAGES`] pages so a key with a very deep
|
||||
/// history cannot turn one convergence check into an unbounded scan; a
|
||||
/// replica beyond that window reads as missing, which only costs a
|
||||
/// re-PUT (today's behaviour), never a lost object.
|
||||
pub async fn find_version_by_etag(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_etag: &str,
|
||||
) -> Result<Option<String>, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
|
||||
let mut key_marker: Option<String> = None;
|
||||
let mut version_id_marker: Option<String> = None;
|
||||
for _ in 0..FIND_VERSION_BY_ETAG_MAX_PAGES {
|
||||
let page = self
|
||||
.client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(object)
|
||||
.max_keys(FIND_VERSION_BY_ETAG_PAGE_SIZE)
|
||||
.set_key_marker(key_marker.take())
|
||||
.set_version_id_marker(version_id_marker.take())
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)?;
|
||||
if let Some(version) = page.versions().iter().find(|version| {
|
||||
version.key() == Some(object)
|
||||
&& version.version_id().is_some_and(|id| !id.is_empty())
|
||||
&& replication_etags_match(Some(source_etag), version.e_tag())
|
||||
}) {
|
||||
return Ok(version.version_id().map(str::to_string));
|
||||
}
|
||||
// Every listed key is >= the prefix; once the listing moved past
|
||||
// the exact key there is nothing left to find.
|
||||
if page
|
||||
.versions()
|
||||
.iter()
|
||||
.any(|version| version.key().is_some_and(|key| key > object))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if !page.is_truncated().unwrap_or(false) {
|
||||
return Ok(None);
|
||||
}
|
||||
key_marker = page.next_key_marker().map(str::to_string);
|
||||
version_id_marker = page.next_version_id_marker().map(str::to_string);
|
||||
if key_marker.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
@@ -2064,7 +2199,15 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
match builder
|
||||
// A forwarded source checksum is this PUT's integrity header. In
|
||||
// streaming-checksum mode (`RUSTFS_REPLICATION_STREAMING_CHECKSUMS`)
|
||||
// the SDK would still add its default CRC32 trailer, and a target that
|
||||
// receives both keeps the trailer's algorithm: a forwarded SHA256
|
||||
// vanished from the replica while the source reported COMPLETED. Pin
|
||||
// this request to WhenRequired so nothing is sent beside the source's
|
||||
// own checksum.
|
||||
let forwards_source_checksum = headers.keys().any(|name| name.as_str().starts_with("x-amz-checksum-"));
|
||||
let mut operation = builder
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.content_length(size)
|
||||
@@ -2084,10 +2227,14 @@ impl TargetClient {
|
||||
}
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
});
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
}
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
// of the stored plaintext, so it cannot be compared against the
|
||||
@@ -2507,13 +2654,21 @@ mod tests {
|
||||
}
|
||||
|
||||
fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
|
||||
header_recording_target_client_with_checksums(response_headers, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn header_recording_target_client_with_checksums(
|
||||
response_headers: Vec<(String, String)>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> (TargetClient, RecordedHeaders) {
|
||||
let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
response_headers,
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
let client =
|
||||
s3_client_for_endpoint_test_with_checksums("https://localhost:443".to_string(), Some(http_client), checksums);
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
@@ -2680,6 +2835,47 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// With streaming checksums enabled the SDK adds a CRC32 trailer to every
|
||||
/// upload. A PUT that forwards the source's checksum must not get that
|
||||
/// second algorithm: a target that receives both keeps the trailer's and
|
||||
/// the forwarded SHA256 never reaches the replica (rustfs/backlog#2340).
|
||||
#[tokio::test]
|
||||
async fn streaming_put_object_with_forwarded_checksum_sends_no_sdk_checksum() {
|
||||
let (client, recorded) =
|
||||
header_recording_target_client_with_checksums(Vec::new(), RequestChecksumCalculation::WhenSupported);
|
||||
let mut forwarded = PutObjectOptions::default();
|
||||
forwarded.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_string(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_string(),
|
||||
);
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &forwarded)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
|
||||
let with_forwarded = &recorded[0];
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-checksum-sha256"),
|
||||
Some("OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=")
|
||||
);
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-trailer"),
|
||||
None,
|
||||
"the SDK must not add a trailer checksum"
|
||||
);
|
||||
assert_eq!(recorded_header(with_forwarded, "x-amz-sdk-checksum-algorithm"), None);
|
||||
// Control: the same client still streams a trailer when nothing is forwarded.
|
||||
let without_forwarded = &recorded[1];
|
||||
assert!(
|
||||
recorded_header(without_forwarded, "x-amz-trailer").is_some(),
|
||||
"streaming mode must still apply to uploads without a forwarded checksum: {without_forwarded:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A forwarded source checksum already satisfies the rule; nothing is added.
|
||||
#[tokio::test]
|
||||
async fn locked_put_object_keeps_a_forwarded_source_checksum() {
|
||||
@@ -3045,6 +3241,14 @@ mod tests {
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test(endpoint: String, http_client: Option<SharedHttpClient>) -> S3Client {
|
||||
s3_client_for_endpoint_test_with_checksums(endpoint, http_client, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test_with_checksums(
|
||||
endpoint: String,
|
||||
http_client: Option<SharedHttpClient>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> S3Client {
|
||||
let credentials = SdkCredentials::builder()
|
||||
.access_key_id("test-access")
|
||||
.secret_access_key("test-secret")
|
||||
@@ -3058,7 +3262,7 @@ mod tests {
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
// Mirror the production remote-target builder so recorded requests
|
||||
// exercise the same checksum/framing behavior (#6853).
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
.request_checksum_calculation(checksums);
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
@@ -3221,6 +3425,64 @@ mod tests {
|
||||
assert!(message.contains("connection refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_replication_service_ignores_resync_and_bandwidth_edits() {
|
||||
let base = BucketTarget {
|
||||
endpoint: "target.example:9000".to_string(),
|
||||
target_bucket: "replica".to_string(),
|
||||
secure: true,
|
||||
path: "on".to_string(),
|
||||
arn: "arn:rustfs:replication:us-east-1:bucket:same".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let resync_edit = BucketTarget {
|
||||
reset_id: "reset-1".to_string(),
|
||||
bandwidth_limit: 1024,
|
||||
..base.clone()
|
||||
};
|
||||
assert!(same_replication_service(&resync_edit, &base));
|
||||
for moved in [
|
||||
BucketTarget {
|
||||
endpoint: "other.example:9000".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
target_bucket: "other".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
secure: false,
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
credentials: Some(Credentials {
|
||||
access_key: "rotated".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..base.clone()
|
||||
},
|
||||
] {
|
||||
assert!(!same_replication_service(&moved, &base));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_verdict_is_per_arn_and_forgotten_with_the_target() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:identity";
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
sys.record_version_identity_capability(arn, VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability("other"), VersionIdentityCapability::Unknown);
|
||||
// A rebuilt target may point at a different service.
|
||||
sys.forget_version_identity_capability(arn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_health_key_preserves_explicit_port() {
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
|
||||
@@ -32,6 +32,7 @@ use crate::bucket::lifecycle::manual_transition_job::{
|
||||
record_manual_transition_worker_result_with_reason, renew_manual_transition_job_lease_if_owned,
|
||||
save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, update_manual_transition_job_record,
|
||||
};
|
||||
use crate::bucket::lifecycle::recovery_disposition_runtime::run_recovery_disposition_maintenance_loop;
|
||||
use crate::bucket::lifecycle::replication_sink;
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, ReplicationStatusType, replication_state_to_filemeta,
|
||||
@@ -149,6 +150,7 @@ pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
static XXHASH_SEED: u64 = 0;
|
||||
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static RECOVERY_DISPOSITION_MAINTENANCE_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
@@ -2398,9 +2400,20 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
|
||||
spawn_tier_delete_journal_recovery_once(api.clone());
|
||||
spawn_transition_transaction_recovery_once(api.clone());
|
||||
spawn_recovery_disposition_maintenance_once(api.clone());
|
||||
spawn_manual_transition_job_recovery_once(api);
|
||||
}
|
||||
|
||||
fn spawn_recovery_disposition_maintenance_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
let cancel_token = api.ctx.background_cancel_token()?;
|
||||
if RECOVERY_DISPOSITION_MAINTENANCE_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
run_recovery_disposition_maintenance_loop(api, cancel_token).await;
|
||||
}))
|
||||
}
|
||||
|
||||
fn spawn_manual_transition_job_recovery_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
if MANUAL_TRANSITION_JOB_RECOVERY_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
|
||||
@@ -41,6 +41,21 @@ where
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -56,6 +71,26 @@ where
|
||||
com::read_config_with_metadata(api, file, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -126,20 +161,30 @@ where
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
delete_config_if_match_with_opts(api, file, etag, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config_if_match_with_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
etag: &str,
|
||||
mut options: ObjectOptions,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
options.http_preconditions = Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
match api.delete_object(RUSTFS_META_BUCKET, file, options).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
|
||||
},
|
||||
manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
|
||||
manual_transition_job, recovery_control, recovery_disposition, recovery_export, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
@@ -42,6 +42,8 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
ManualTransitionTask,
|
||||
ManualTransitionWorkerResult,
|
||||
RecoveryControl,
|
||||
RecoveryExport,
|
||||
RecoveryDisposition,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -112,8 +114,20 @@ pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNam
|
||||
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryControl,
|
||||
};
|
||||
pub(crate) const RECOVERY_EXPORT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-export",
|
||||
prefix: recovery_export::ILM_RECOVERY_EXPORT_PREFIX,
|
||||
max_record_size: recovery_export::MAX_ILM_RECOVERY_EXPORT_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryExport,
|
||||
};
|
||||
pub(crate) const RECOVERY_DISPOSITION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-disposition",
|
||||
prefix: recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
max_record_size: recovery_disposition::MAX_ILM_RECOVERY_DISPOSITION_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryDisposition,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
@@ -124,6 +138,8 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
RECOVERY_CONTROL_NAMESPACE,
|
||||
RECOVERY_EXPORT_NAMESPACE,
|
||||
RECOVERY_DISPOSITION_NAMESPACE,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -261,6 +277,28 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
owner_fence_sha256: Option<String>,
|
||||
},
|
||||
RecoveryExport {
|
||||
content_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
topology_generation: String,
|
||||
member_epochs_sha256: String,
|
||||
creator_sha256: String,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
RecoveryDisposition {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
copy_manifest_sha256: String,
|
||||
copy_manifest_count: usize,
|
||||
created_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence_sha256: Option<String>,
|
||||
owner_lease_acquired_at_unix_nanos: Option<i64>,
|
||||
owner_lease_expires_at_unix_nanos: Option<i64>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
}
|
||||
|
||||
impl DurableIlmRecordCheckpoint {
|
||||
@@ -275,7 +313,9 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 }
|
||||
| Self::RecoveryControl { content_sha256, .. } => content_sha256,
|
||||
| Self::RecoveryControl { content_sha256, .. }
|
||||
| Self::RecoveryExport { content_sha256, .. }
|
||||
| Self::RecoveryDisposition { content_sha256, .. } => content_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -316,6 +356,9 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
|
||||
}
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return Err(Error::other("durable ILM recovery disposition checkpoint is invalid"));
|
||||
}
|
||||
}
|
||||
if self == next {
|
||||
if let Self::ManualTransitionJob {
|
||||
@@ -456,9 +499,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
},
|
||||
) => {
|
||||
previous_identity == next_identity
|
||||
&& transition_state_distance(*previous_state, *next_state)
|
||||
.and_then(|distance| previous_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| *next_revision == expected_revision)
|
||||
&& transition_state_revision_is_successor(*previous_state, *previous_revision, *next_state, *next_revision)
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
@@ -594,6 +635,83 @@ impl DurableIlmRecordCheckpoint {
|
||||
&& previous_attempts == next_attempts;
|
||||
adjacent && (claim || source_refresh || completion)
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
owner_lease_acquired_at_unix_nanos: previous_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: previous_owner_expires,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: next_identity,
|
||||
copy_manifest_sha256: next_manifest,
|
||||
copy_manifest_count: next_manifest_count,
|
||||
created_at_unix_nanos: next_created_at,
|
||||
revision: next_revision,
|
||||
state: next_state,
|
||||
owner_fence_sha256: next_owner,
|
||||
owner_lease_acquired_at_unix_nanos: next_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: next_owner_expires,
|
||||
confirmed_absent_sha256: next_confirmed,
|
||||
retain_until_unix_nanos: next_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let immutable_identity_matches = previous_identity == next_identity
|
||||
&& previous_manifest == next_manifest
|
||||
&& previous_manifest_count == next_manifest_count
|
||||
&& previous_created_at == next_created_at
|
||||
&& previous_retain_until == next_retain_until;
|
||||
let adjacent = previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let progress_is_monotonic = sorted_sha256_set_is_subset(previous_confirmed, next_confirmed);
|
||||
let legal_edge = match (previous_state, next_state) {
|
||||
(Prepared, Prepared) => {
|
||||
let claim = previous_owner.is_none() && next_owner.is_some();
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
previous_confirmed == next_confirmed && (claim || takeover)
|
||||
}
|
||||
(Prepared, Applying) => {
|
||||
previous_confirmed == next_confirmed
|
||||
&& previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
}
|
||||
(Applying, Applying) => {
|
||||
let progress = previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
&& previous_confirmed.len().checked_add(1) == Some(next_confirmed.len());
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_confirmed == next_confirmed
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
progress || takeover
|
||||
}
|
||||
(Applying, Completed) => {
|
||||
previous_owner.is_some() && next_owner.is_none() && previous_confirmed == next_confirmed
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
immutable_identity_matches && adjacent && progress_is_monotonic && legal_edge
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
@@ -611,6 +729,11 @@ impl DurableIlmRecordCheckpoint {
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
for checkpoint in [self, terminal] {
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Self::TierProbeIntent { state, .. } = terminal
|
||||
&& !matches!(
|
||||
state,
|
||||
@@ -627,6 +750,11 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryDisposition { state, .. } = terminal
|
||||
&& state != &recovery_disposition::IlmRecoveryDispositionState::Completed
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -752,11 +880,121 @@ impl DurableIlmRecordCheckpoint {
|
||||
&& terminal_revision > previous_revision
|
||||
&& terminal_attempts >= previous_attempts
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: terminal_identity,
|
||||
copy_manifest_sha256: terminal_manifest,
|
||||
copy_manifest_count: terminal_manifest_count,
|
||||
created_at_unix_nanos: terminal_created_at,
|
||||
revision: terminal_revision,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState::Completed,
|
||||
confirmed_absent_sha256: terminal_confirmed,
|
||||
retain_until_unix_nanos: terminal_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
matches!(
|
||||
previous_state,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared
|
||||
| recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
) && previous_identity == terminal_identity
|
||||
&& previous_manifest == terminal_manifest
|
||||
&& previous_manifest_count == terminal_manifest_count
|
||||
&& previous_created_at == terminal_created_at
|
||||
&& previous_retain_until == terminal_retain_until
|
||||
&& terminal_revision.checked_sub(*previous_revision).is_some_and(|distance| {
|
||||
let minimum_distance = match previous_state {
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared if previous_owner.is_some() => 3,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared => 4,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
if previous_confirmed.len() == *previous_manifest_count =>
|
||||
{
|
||||
1
|
||||
}
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying => 2,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Completed => u64::MAX,
|
||||
};
|
||||
distance >= minimum_distance
|
||||
})
|
||||
&& sorted_sha256_set_is_subset(previous_confirmed, terminal_confirmed)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint_is_valid(checkpoint: &DurableIlmRecordCheckpoint) -> bool {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
copy_manifest_sha256,
|
||||
copy_manifest_count,
|
||||
created_at_unix_nanos,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos,
|
||||
} = checkpoint
|
||||
else {
|
||||
return true;
|
||||
};
|
||||
let owner_fence_sha256 = owner_fence_sha256.as_deref();
|
||||
|
||||
is_canonical_sha256(content_sha256)
|
||||
&& is_canonical_sha256(identity_sha256)
|
||||
&& is_canonical_sha256(copy_manifest_sha256)
|
||||
&& *copy_manifest_count > 0
|
||||
&& *created_at_unix_nanos > 0
|
||||
&& *revision > 0
|
||||
&& *retain_until_unix_nanos > 0
|
||||
&& owner_fence_sha256.is_none_or(is_canonical_sha256)
|
||||
&& match (
|
||||
owner_fence_sha256,
|
||||
*owner_lease_acquired_at_unix_nanos,
|
||||
*owner_lease_expires_at_unix_nanos,
|
||||
) {
|
||||
(None, None, None) => true,
|
||||
(Some(_), Some(acquired), Some(expires)) => acquired > 0 && expires > acquired,
|
||||
_ => false,
|
||||
}
|
||||
&& confirmed_absent_sha256.len() <= *copy_manifest_count
|
||||
&& confirmed_absent_sha256.iter().all(|digest| is_canonical_sha256(digest))
|
||||
&& confirmed_absent_sha256.windows(2).all(|pair| pair[0] < pair[1])
|
||||
&& match *state {
|
||||
Prepared => confirmed_absent_sha256.is_empty(),
|
||||
Applying => owner_fence_sha256.is_some(),
|
||||
Completed => owner_fence_sha256.is_none() && confirmed_absent_sha256.len() == *copy_manifest_count,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_canonical_sha256(value: &str) -> bool {
|
||||
is_sha256_checksum(value)
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn sorted_sha256_set_is_subset(subset: &[String], superset: &[String]) -> bool {
|
||||
subset.iter().all(|candidate| superset.binary_search(candidate).is_ok())
|
||||
}
|
||||
|
||||
fn tier_delete_dispatch_parent_progress_delta(
|
||||
previous_sequence: u64,
|
||||
previous_completed_journals: u64,
|
||||
@@ -790,6 +1028,22 @@ fn transition_state_distance(
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_state_revision_is_successor(
|
||||
from: transition_transaction::TransitionTransactionState,
|
||||
from_revision: u64,
|
||||
to: transition_transaction::TransitionTransactionState,
|
||||
to_revision: u64,
|
||||
) -> bool {
|
||||
use transition_transaction::TransitionTransactionState::{LocalCommitStarted, UploadOutcomeUnknown};
|
||||
|
||||
if from == UploadOutcomeUnknown && from_revision == 1 && to == LocalCommitStarted {
|
||||
return to_revision == 2;
|
||||
}
|
||||
transition_state_distance(from, to)
|
||||
.and_then(|distance| from_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| to_revision == expected_revision)
|
||||
}
|
||||
|
||||
fn tier_probe_state_reaches(
|
||||
from: tier_probe_intent::TierProbeIntentState,
|
||||
to: tier_probe_intent::TierProbeIntentState,
|
||||
@@ -1348,6 +1602,55 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryExport => {
|
||||
let (protocol, export_id) = recovery_export::recovery_export_id_from_record_object_name(path)?;
|
||||
let export = recovery_export::IlmRecoveryExport::decode(&export_id, data)?;
|
||||
let canonical = recovery_export::recovery_export_record_object_name(protocol, &export_id)?;
|
||||
if canonical != path || export.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
let source_generation_sha256 = checkpoint_hash(&export.source_generation)?;
|
||||
(
|
||||
"export_id",
|
||||
export_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryExport {
|
||||
content_sha256,
|
||||
source_generation_sha256,
|
||||
topology_generation: export.topology_generation,
|
||||
member_epochs_sha256: export.member_epochs_sha256,
|
||||
creator_sha256: export.creator_sha256,
|
||||
retain_until_unix_nanos: export.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryDisposition => {
|
||||
// The disposition module owns strict schema, checksum, canonical
|
||||
// path, immutable-manifest, and state-specific validation. Keep
|
||||
// this boundary limited to decommission identity/checkpoint
|
||||
// projection so the two readers cannot accept different records.
|
||||
let disposition = recovery_disposition::decode_recovery_disposition_checkpoint(path, data)?;
|
||||
if disposition.content_sha256 != content_sha256 {
|
||||
return Err(Error::other("ILM recovery disposition checkpoint content digest is invalid"));
|
||||
}
|
||||
(
|
||||
"disposition_id",
|
||||
disposition.disposition_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: disposition.content_sha256,
|
||||
identity_sha256: disposition.identity_sha256,
|
||||
copy_manifest_sha256: disposition.copy_manifest_sha256,
|
||||
copy_manifest_count: disposition.copy_manifest_count,
|
||||
created_at_unix_nanos: disposition.created_at_unix_nanos,
|
||||
revision: disposition.revision,
|
||||
state: disposition.state,
|
||||
owner_fence_sha256: disposition.owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos: disposition.owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos: disposition.owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256: disposition.confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: disposition.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1503,6 +1806,203 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint(
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence: Option<&str>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
) -> DurableIlmRecordCheckpoint {
|
||||
let (owner_lease_acquired_at_unix_nanos, owner_lease_expires_at_unix_nanos) = match owner_fence {
|
||||
Some("f") => (Some(10), Some(20)),
|
||||
Some(_) => (Some(1), Some(10)),
|
||||
None => (None, None),
|
||||
};
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: "a".repeat(64),
|
||||
copy_manifest_sha256: "d".repeat(64),
|
||||
copy_manifest_count: 2,
|
||||
created_at_unix_nanos: 1_700_000_000_000_000_000,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256: owner_fence.map(|digest| digest.repeat(64)),
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: 1_820_000_000_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_namespace_is_registered_without_shadowing_its_root() {
|
||||
let disposition_id = "a".repeat(64);
|
||||
let path = format!(
|
||||
"{}/tier_delete_journal/{}/{}/{}.json",
|
||||
recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
&disposition_id[..2],
|
||||
&disposition_id[2..4],
|
||||
disposition_id
|
||||
);
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery disposition path should classify")
|
||||
.expect("recovery disposition should be durable");
|
||||
|
||||
assert_eq!(namespace, &RECOVERY_DISPOSITION_NAMESPACE);
|
||||
assert!(classify_durable_ilm_record(recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_checkpoint_accepts_only_monotonic_progress() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let claimed = recovery_disposition_checkpoint(2, Prepared, Some("e"), Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), Vec::new());
|
||||
let first_absent = recovery_disposition_checkpoint(4, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let taken_over = recovery_disposition_checkpoint(5, Applying, Some("f"), vec![first_copy.clone()]);
|
||||
let all_absent = recovery_disposition_checkpoint(6, Applying, Some("f"), vec![first_copy.clone(), second_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(7, Completed, None, vec![first_copy.clone(), second_copy.clone()]);
|
||||
|
||||
prepared
|
||||
.validate_successor(&claimed)
|
||||
.expect("Prepared should record an owner claim without absence progress");
|
||||
claimed
|
||||
.validate_successor(&applying)
|
||||
.expect("Prepared should advance to Applying without folding in deletion progress");
|
||||
applying
|
||||
.validate_successor(&first_absent)
|
||||
.expect("Applying should append newly confirmed absent copies");
|
||||
first_absent
|
||||
.validate_successor(&taken_over)
|
||||
.expect("Applying should record a fenced owner takeover without losing progress");
|
||||
taken_over
|
||||
.validate_successor(&all_absent)
|
||||
.expect("Applying should preserve every earlier confirmation while making progress");
|
||||
all_absent
|
||||
.validate_successor(&completed)
|
||||
.expect("a fully confirmed manifest should advance to Completed");
|
||||
|
||||
assert!(
|
||||
prepared.validate_successor(&completed).is_err(),
|
||||
"adjacent receipt updates must not skip Applying"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), Vec::new()))
|
||||
.is_err(),
|
||||
"confirmed-absent progress must not move backwards"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Completed, None, vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"Completed must cover the complete immutable copy manifest"
|
||||
);
|
||||
assert!(
|
||||
completed
|
||||
.validate_successor(&recovery_disposition_checkpoint(7, Applying, Some("e"), vec![second_copy]))
|
||||
.is_err(),
|
||||
"Completed is terminal"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"a same-state revision bump must change the owner fence or absence progress"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Applying, None, vec![first_copy]))
|
||||
.is_err(),
|
||||
"Applying must retain a fenced owner"
|
||||
);
|
||||
|
||||
let mut noncanonical_identity = claimed.clone();
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut noncanonical_identity {
|
||||
*identity_sha256 = "A".repeat(64);
|
||||
}
|
||||
assert!(prepared.validate_successor(&noncanonical_identity).is_err());
|
||||
let mut changed_created_at = claimed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
created_at_unix_nanos, ..
|
||||
} = &mut changed_created_at
|
||||
{
|
||||
*created_at_unix_nanos += 1;
|
||||
}
|
||||
assert!(prepared.validate_successor(&changed_created_at).is_err());
|
||||
|
||||
let mut early_takeover = taken_over;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
..
|
||||
} = &mut early_takeover
|
||||
{
|
||||
*owner_lease_acquired_at_unix_nanos = Some(9);
|
||||
}
|
||||
assert!(first_absent.validate_successor(&early_takeover).is_err());
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Applying,
|
||||
Some("e"),
|
||||
vec!["c".repeat(64), "b".repeat(64)],
|
||||
))
|
||||
.is_err(),
|
||||
"confirmed-absent entries must be a canonical sorted set"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_terminal_predecessor_requires_exact_identity_and_full_manifest() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(5, Completed, None, vec![first_copy.clone(), second_copy]);
|
||||
|
||||
assert!(prepared.is_predecessor_of_terminal(&completed));
|
||||
assert!(applying.is_predecessor_of_terminal(&completed));
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(2, Applying, Some("e"), Vec::new())),
|
||||
"a nonterminal disposition must not authorize terminal cleanup"
|
||||
);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"terminal proof must leave enough revisions for claim, apply, progress, and completion"
|
||||
);
|
||||
assert!(
|
||||
!applying.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"an incomplete Applying checkpoint cannot complete without a progress generation"
|
||||
);
|
||||
|
||||
let mut other_identity = completed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut other_identity {
|
||||
*identity_sha256 = "e".repeat(64);
|
||||
}
|
||||
assert!(!prepared.is_predecessor_of_terminal(&other_identity));
|
||||
|
||||
let incomplete_terminal = recovery_disposition_checkpoint(4, Completed, None, vec![first_copy]);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&incomplete_terminal),
|
||||
"a partial confirmed-absent set must not become terminal proof"
|
||||
);
|
||||
}
|
||||
|
||||
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
|
||||
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
tier_probe_intent::TierProbeIntent {
|
||||
@@ -1687,6 +2187,64 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_checkpoint_accepts_only_the_distinguishable_compact_edge() {
|
||||
let identity_sha256 = "a".repeat(64);
|
||||
let unknown_remote_sha256 = "b".repeat(64);
|
||||
let known_remote_sha256 = "c".repeat(64);
|
||||
let checkpoint = |revision, state, remote_version_sha256: String, remote_version_known| {
|
||||
DurableIlmRecordCheckpoint::TransitionTransaction {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: identity_sha256.clone(),
|
||||
remote_version_sha256,
|
||||
remote_version_known,
|
||||
revision,
|
||||
state,
|
||||
}
|
||||
};
|
||||
let compact_unknown = checkpoint(
|
||||
1,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256.clone(),
|
||||
false,
|
||||
);
|
||||
let compact_local_commit = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
compact_unknown
|
||||
.validate_successor(&compact_local_commit)
|
||||
.expect("compact pre-upload fence should advance directly to the exact local-commit fence");
|
||||
|
||||
let legacy_unknown = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256,
|
||||
false,
|
||||
);
|
||||
let invalid_legacy_skip = checkpoint(
|
||||
3,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
assert!(
|
||||
legacy_unknown.validate_successor(&invalid_legacy_skip).is_err(),
|
||||
"legacy UploadOutcomeUnknown@2 must not masquerade as the compact edge"
|
||||
);
|
||||
let valid_legacy_skip = checkpoint(
|
||||
4,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256,
|
||||
true,
|
||||
);
|
||||
legacy_unknown
|
||||
.validate_successor(&valid_legacy_skip)
|
||||
.expect("legacy receipts may still observe the existing two-edge state advance");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
@@ -25,6 +25,9 @@ mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod recovery_control;
|
||||
pub mod recovery_disposition;
|
||||
pub(crate) mod recovery_disposition_runtime;
|
||||
pub mod recovery_export;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
|
||||
@@ -168,7 +168,7 @@ impl IlmRecoverySourceGeneration {
|
||||
Ok(generation)
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<()> {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
if self.source_schema.trim().is_empty() {
|
||||
return Err(IlmRecoveryControlError::Corrupt("source schema is empty"));
|
||||
}
|
||||
@@ -485,6 +485,40 @@ impl IlmRecoveryControl {
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn abandon_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> {
|
||||
if self.owner.is_some()
|
||||
|| self.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| &self.observed_source_generation != expected_source_generation
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator abandonment requires the exact ownerless retained source generation",
|
||||
));
|
||||
}
|
||||
self.bump_revision()?;
|
||||
self.classification = IlmRecoveryClassification::Abandoned;
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn retry_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> {
|
||||
if self.owner.is_some()
|
||||
|| !matches!(
|
||||
self.classification,
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired
|
||||
)
|
||||
|| self.attempt_count == u64::MAX
|
||||
|| &self.observed_source_generation != expected_source_generation
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator retry requires the exact ownerless retained source generation",
|
||||
));
|
||||
}
|
||||
self.bump_revision()?;
|
||||
self.classification = IlmRecoveryClassification::Retrying;
|
||||
self.consecutive_failure_count = 0;
|
||||
self.next_attempt_at_unix_nanos = None;
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn validate_successor(&self, next: &Self) -> Result<()> {
|
||||
self.validate()?;
|
||||
next.validate()?;
|
||||
@@ -508,12 +542,58 @@ impl IlmRecoveryControl {
|
||||
self.validate_failure_successor(next)
|
||||
}
|
||||
(Some(_), None) => self.validate_finish_successor(next),
|
||||
(None, None)
|
||||
if self.classification == IlmRecoveryClassification::RetainedAmbiguous
|
||||
&& next.classification == IlmRecoveryClassification::Abandoned =>
|
||||
{
|
||||
self.validate_operator_abandon_successor(next)
|
||||
}
|
||||
(None, None)
|
||||
if matches!(
|
||||
self.classification,
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired
|
||||
) && next.classification == IlmRecoveryClassification::Retrying =>
|
||||
{
|
||||
self.validate_operator_retry_successor(next)
|
||||
}
|
||||
(None, None) => Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"ownerless control cannot advance without a claim",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_operator_abandon_successor(&self, next: &Self) -> Result<()> {
|
||||
if next.observed_source_generation != self.observed_source_generation
|
||||
|| next.attempt_count != self.attempt_count
|
||||
|| next.consecutive_failure_count != self.consecutive_failure_count
|
||||
|| next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos
|
||||
|| next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos
|
||||
|| next.next_attempt_at_unix_nanos != self.next_attempt_at_unix_nanos
|
||||
|| next.last_error_code != self.last_error_code
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator abandonment changed recovery history or source generation",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_operator_retry_successor(&self, next: &Self) -> Result<()> {
|
||||
if next.observed_source_generation != self.observed_source_generation
|
||||
|| next.attempt_count != self.attempt_count
|
||||
|| next.consecutive_failure_count != 0
|
||||
|| next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos
|
||||
|| next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos
|
||||
|| next.next_attempt_at_unix_nanos.is_some()
|
||||
|| next.last_error_code != self.last_error_code
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator retry changed recovery history or source generation",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_claim_successor(&self, next: &Self) -> Result<()> {
|
||||
if self.classification != IlmRecoveryClassification::Retrying
|
||||
|| next.classification != IlmRecoveryClassification::Retrying
|
||||
@@ -827,6 +907,23 @@ pub async fn observe_recovery_source(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
observe_recovery_source_with_options(api, canonical_path, source_schema, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn observe_recovery_source_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
observe_recovery_source_with_options(api, canonical_path, source_schema, true).await
|
||||
}
|
||||
|
||||
async fn observe_recovery_source_with_options(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
no_lock: bool,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
validate_canonical_source_path(canonical_path).map_err(recovery_control_store_error)?;
|
||||
if source_schema.trim().is_empty() {
|
||||
@@ -837,7 +934,16 @@ pub async fn observe_recovery_source(
|
||||
let mut observations = Vec::new();
|
||||
for set in api.all_set_disks() {
|
||||
let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index);
|
||||
match config_boundary::read_config_with_metadata(set, canonical_path, &ObjectOptions::default()).await {
|
||||
match config_boundary::read_config_with_metadata(
|
||||
set,
|
||||
canonical_path,
|
||||
&ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((data, metadata)) => {
|
||||
let etag = metadata
|
||||
.etag
|
||||
@@ -1255,6 +1361,99 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_abandonment_is_an_exact_ownerless_retained_successor() {
|
||||
let mut retained = IlmRecoveryControl::new(
|
||||
control().identity,
|
||||
generation(),
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
1_000_000_000,
|
||||
IlmRecoveryErrorCode::OperatorDispositionRequired,
|
||||
)
|
||||
.expect("retained control should build");
|
||||
let previous = retained.clone();
|
||||
retained
|
||||
.abandon_for_operator(&previous.observed_source_generation)
|
||||
.expect("exact retained generation should be abandonable");
|
||||
previous
|
||||
.validate_successor(&retained)
|
||||
.expect("operator abandonment should be a valid successor");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::Abandoned);
|
||||
assert_eq!(retained.revision, previous.revision + 1);
|
||||
|
||||
let mut wrong_generation = previous.clone();
|
||||
let mut generation = previous.observed_source_generation.clone();
|
||||
generation.source_etag = "different".to_string();
|
||||
assert!(wrong_generation.abandon_for_operator(&generation).is_err());
|
||||
|
||||
let mut mutated_history = retained.clone();
|
||||
mutated_history.attempt_count += 1;
|
||||
assert!(previous.validate_successor(&mutated_history).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_retry_rearms_exact_retained_generation_without_resetting_history() {
|
||||
for classification in [
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
IlmRecoveryClassification::OperatorRequired,
|
||||
] {
|
||||
let mut retained = control();
|
||||
retained
|
||||
.claim("node-a", Uuid::new_v4(), 2_000_000_000, 1)
|
||||
.expect("attempt should claim");
|
||||
retained
|
||||
.record_retryable_failure(2_000_000_001, IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("failure should persist");
|
||||
retained.classification = classification;
|
||||
retained.next_attempt_at_unix_nanos = None;
|
||||
if classification == IlmRecoveryClassification::OperatorRequired {
|
||||
retained.attempt_count = u64::from(MAX_RECOVERY_ATTEMPTS);
|
||||
retained.consecutive_failure_count = MAX_RECOVERY_ATTEMPTS;
|
||||
}
|
||||
retained.validate().expect("retained control should remain valid");
|
||||
|
||||
let previous = retained.clone();
|
||||
retained
|
||||
.retry_for_operator(&previous.observed_source_generation)
|
||||
.expect("exact retained generation should be retryable");
|
||||
previous
|
||||
.validate_successor(&retained)
|
||||
.expect("operator retry should be a valid successor");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(retained.revision, previous.revision + 1);
|
||||
assert_eq!(retained.attempt_count, previous.attempt_count);
|
||||
assert_eq!(retained.first_failure_at_unix_nanos, previous.first_failure_at_unix_nanos);
|
||||
assert_eq!(retained.last_failure_at_unix_nanos, previous.last_failure_at_unix_nanos);
|
||||
assert_eq!(retained.last_error_code, previous.last_error_code);
|
||||
assert_eq!(retained.consecutive_failure_count, 0);
|
||||
assert_eq!(retained.next_attempt_at_unix_nanos, None);
|
||||
assert!(retained.should_attempt_at(2_000_000_002));
|
||||
|
||||
if classification == IlmRecoveryClassification::OperatorRequired {
|
||||
retained
|
||||
.claim("node-b", Uuid::new_v4(), 2_000_000_002, 1)
|
||||
.expect("operator retry should authorize one new bounded attempt");
|
||||
retained
|
||||
.record_retryable_failure(2_000_000_003, IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("the bounded attempt failure should persist");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
assert_eq!(retained.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS) + 1);
|
||||
assert_eq!(retained.consecutive_failure_count, 1);
|
||||
}
|
||||
|
||||
let mut wrong_generation = previous;
|
||||
let mut changed_generation = wrong_generation.observed_source_generation.clone();
|
||||
changed_generation.source_etag = "changed".to_string();
|
||||
assert!(wrong_generation.retry_for_operator(&changed_generation).is_err());
|
||||
}
|
||||
|
||||
let mut exhausted = control();
|
||||
exhausted.classification = IlmRecoveryClassification::OperatorRequired;
|
||||
exhausted.attempt_count = u64::MAX;
|
||||
let generation = exhausted.observed_source_generation.clone();
|
||||
assert!(exhausted.retry_for_operator(&generation).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_control_view_redacts_source_and_owner_details() {
|
||||
let mut control = control();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,840 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::config_boundary;
|
||||
use super::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryProtocol, IlmRecoverySourceCopy, IlmRecoverySourceGeneration,
|
||||
MAX_ILM_RECOVERY_CONTROL_SIZE, ObservedIlmRecoveryControl, ObservedIlmRecoverySource, recovery_control_record_object_name,
|
||||
};
|
||||
use super::tier_delete_journal::{
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_source,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{ObjectOptions, WriteCompletion};
|
||||
use crate::services::notification_sys::{
|
||||
acquire_ilm_recovery_export_fleet_proof, ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_member_epochs_sha256,
|
||||
ilm_recovery_export_topology_generation,
|
||||
};
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, namespace::NamespaceLocking as _, object::HTTPPreconditions};
|
||||
use crate::store::ECStore;
|
||||
|
||||
pub const ILM_RECOVERY_EXPORT_SCHEMA: &str = "rustfs-ilm-recovery-export-v1";
|
||||
pub const ILM_RECOVERY_EXPORT_PREFIX: &str = "ilm/recovery-exports";
|
||||
pub const MAX_ILM_RECOVERY_EXPORT_SIZE: usize = 128 * 1024;
|
||||
const MAX_ILM_RECOVERY_EXPORTS: usize = 10_000;
|
||||
const MAX_ILM_RECOVERY_EXPORT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
const MAX_ACTOR_EXPORTS_PER_MINUTE: usize = 10;
|
||||
const MAX_CLUSTER_EXPORTS_PER_MINUTE: usize = 100;
|
||||
const EXPORT_RETENTION_NANOS: i64 = 90 * 24 * 60 * 60 * 1_000_000_000;
|
||||
const EXPORT_ADMISSION_LOCK: &str = "ilm/recovery-admission/export.lock";
|
||||
const MAX_LEGACY_TIER_DELETE_SOURCE_SIZE: usize = 64 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExportObservation {
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExport {
|
||||
pub export_id: String,
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
pub creator_sha256: String,
|
||||
pub created_at_unix_nanos: i64,
|
||||
pub retain_until_unix_nanos: i64,
|
||||
pub source_bytes_base64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedIlmRecoveryExport {
|
||||
schema: String,
|
||||
content_sha256: String,
|
||||
export: IlmRecoveryExport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct IlmRecoveryExportCreated {
|
||||
pub export_id: String,
|
||||
pub content_sha256: String,
|
||||
pub encoded: Vec<u8>,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExport {
|
||||
fn validate(&self) -> Result<()> {
|
||||
self.source_generation.validate().map_err(Error::other)?;
|
||||
validate_sha256(&self.export_id, "ILM recovery export ID is invalid")?;
|
||||
validate_sha256(&self.control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&self.topology_generation, "ILM recovery export topology generation is invalid")?;
|
||||
validate_sha256(&self.member_epochs_sha256, "ILM recovery export member epoch digest is invalid")?;
|
||||
validate_sha256(&self.creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
if self.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| self.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&self.source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source is not an exportable legacy journal"));
|
||||
}
|
||||
if self.control_etag.trim().is_empty() || self.control_revision == 0 {
|
||||
return Err(Error::other("ILM recovery export control generation is invalid"));
|
||||
}
|
||||
if self.canonical_source_path.is_empty()
|
||||
|| self.canonical_source_path.starts_with('/')
|
||||
|| self.canonical_source_path.ends_with('/')
|
||||
|| self.canonical_source_path.split('/').any(str::is_empty)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source path is invalid"));
|
||||
}
|
||||
if self.created_at_unix_nanos <= 0
|
||||
|| self.retain_until_unix_nanos < self.created_at_unix_nanos.saturating_add(EXPORT_RETENTION_NANOS)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export retention is invalid"));
|
||||
}
|
||||
let source = base64_simd::STANDARD
|
||||
.decode_to_vec(self.source_bytes_base64.as_bytes())
|
||||
.map_err(|_| Error::other("ILM recovery export source encoding is invalid"))?;
|
||||
validate_legacy_tier_delete_recovery_source(&self.canonical_source_path, &self.source_generation.source_schema, &source)?;
|
||||
let encoded_len = u64::try_from(source.len()).map_err(|_| Error::other("ILM recovery export source length overflow"))?;
|
||||
if source.is_empty()
|
||||
|| source.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE
|
||||
|| hex_sha256(&source, ToOwned::to_owned) != self.source_generation.content_sha256
|
||||
|| self.source_generation.copies.iter().any(|copy| {
|
||||
copy.canonical_path != self.canonical_source_path
|
||||
|| copy.etag != self.source_generation.source_etag
|
||||
|| copy.content_sha256 != self.source_generation.content_sha256
|
||||
|| copy.encoded_len != encoded_len
|
||||
})
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source bytes do not match the observed generation"));
|
||||
}
|
||||
if recovery_export_id(&self.control_id, &self.source_generation)? != self.export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match its source generation"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
let export_bytes = serde_json::to_vec(self).map_err(Error::other)?;
|
||||
let persisted = PersistedIlmRecoveryExport {
|
||||
schema: ILM_RECOVERY_EXPORT_SCHEMA.to_string(),
|
||||
content_sha256: hex_sha256(&export_bytes, ToOwned::to_owned),
|
||||
export: self.clone(),
|
||||
};
|
||||
let encoded = serde_json::to_vec(&persisted).map_err(Error::other)?;
|
||||
if encoded.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
pub fn decode(expected_export_id: &str, data: &[u8]) -> Result<Self> {
|
||||
validate_sha256(expected_export_id, "ILM recovery export ID is invalid")?;
|
||||
if data.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
let persisted: PersistedIlmRecoveryExport = serde_json::from_slice(data).map_err(Error::other)?;
|
||||
if persisted.schema != ILM_RECOVERY_EXPORT_SCHEMA {
|
||||
return Err(Error::other("ILM recovery export schema is unsupported"));
|
||||
}
|
||||
validate_sha256(&persisted.content_sha256, "ILM recovery export checksum is invalid")?;
|
||||
let export_bytes = serde_json::to_vec(&persisted.export).map_err(Error::other)?;
|
||||
if hex_sha256(&export_bytes, ToOwned::to_owned) != persisted.content_sha256 {
|
||||
return Err(Error::other("ILM recovery export checksum mismatch"));
|
||||
}
|
||||
persisted.export.validate()?;
|
||||
if persisted.export.export_id != expected_export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match record key"));
|
||||
}
|
||||
Ok(persisted.export)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recovery_export_record_object_name(protocol: IlmRecoveryProtocol, export_id: &str) -> Result<String> {
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
Ok(format!(
|
||||
"{}/{}/{}/{}/{}.json",
|
||||
ILM_RECOVERY_EXPORT_PREFIX,
|
||||
protocol.as_str(),
|
||||
&export_id[..2],
|
||||
&export_id[2..4],
|
||||
export_id
|
||||
))
|
||||
}
|
||||
|
||||
pub fn recovery_export_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> {
|
||||
let suffix = object
|
||||
.strip_prefix(ILM_RECOVERY_EXPORT_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.ok_or_else(|| Error::other("ILM recovery export path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let protocol = match parts.next() {
|
||||
Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal,
|
||||
_ => return Err(Error::other("ILM recovery export protocol is invalid")),
|
||||
};
|
||||
let shard_a = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let export_id = parts
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.ok_or_else(|| Error::other("ILM recovery export suffix is invalid"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
if shard_a != &export_id[..2] || shard_b != &export_id[2..4] {
|
||||
return Err(Error::other("ILM recovery export shard does not match export ID"));
|
||||
}
|
||||
Ok((protocol, export_id.to_string()))
|
||||
}
|
||||
|
||||
pub async fn inspect_recovery_export_observation(api: Arc<ECStore>, control_id: &str) -> Result<IlmRecoveryExportObservation> {
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
let observed_control = load_exportable_control(api.clone(), control_id).await?;
|
||||
let observed_source = observe_export_source(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(&proof).await
|
||||
{
|
||||
return Err(Error::other("ILM recovery export observation changed or is incomplete"));
|
||||
}
|
||||
Ok(IlmRecoveryExportObservation {
|
||||
control_id: control_id.to_string(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(&proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(&proof),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn create_recovery_export(
|
||||
api: Arc<ECStore>,
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
) -> Result<IlmRecoveryExportCreated> {
|
||||
validate_sha256(creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, EXPORT_ADMISSION_LOCK).await?;
|
||||
let admission_guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
if ilm_recovery_export_topology_generation(&proof) != observation.topology_generation
|
||||
|| ilm_recovery_export_member_epochs_sha256(&proof) != observation.member_epochs_sha256
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.map_err(Error::other)?;
|
||||
let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let control_guard = control_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let source_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &observation.canonical_source_path)
|
||||
.await?;
|
||||
let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let locks_current = || !admission_guard.is_lock_lost() && !control_guard.is_lock_lost() && !source_guard.is_lock_lost();
|
||||
let (current, current_source_bytes) = current_observation_under_proof_no_lock(api.clone(), observation, &proof).await?;
|
||||
if ¤t != observation || !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let current_source_base64 = base64_simd::STANDARD.encode_to_string(current_source_bytes);
|
||||
let candidate_export_id = recovery_export_id(¤t.control_id, ¤t.source_generation)?;
|
||||
let object = recovery_export_record_object_name(current.protocol, &candidate_export_id)?;
|
||||
match load_recovery_export_decoded(api.clone(), &candidate_export_id).await {
|
||||
Ok((existing, export)) if export_matches_observation(&export, observation) => {
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &existing.encoded)
|
||||
.await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
return Ok(existing.with_replayed());
|
||||
}
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let inventory = collect_export_inventory(api.clone()).await?;
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let created_at_unix_nanos = now_unix_nanos()?;
|
||||
let export = build_export_from_source(¤t, creator_sha256, created_at_unix_nanos, ¤t_source_base64)?;
|
||||
let encoded = export.encode()?;
|
||||
inventory.check(creator_sha256, encoded.len(), created_at_unix_nanos)?;
|
||||
|
||||
let mut write_options = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
write_options.add_namespace_lock_guard(&admission_guard);
|
||||
write_options.add_namespace_lock_guard(&control_guard);
|
||||
write_options.add_namespace_lock_guard(&source_guard);
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded.clone(), &write_options).await;
|
||||
let stored = match load_recovery_export(api.clone(), &export.export_id).await {
|
||||
Ok(stored) if stored.encoded == encoded => stored,
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(read_err) => return Err(write_result.err().unwrap_or(read_err)),
|
||||
};
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &encoded).await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
pub async fn load_recovery_export(api: Arc<ECStore>, export_id: &str) -> Result<IlmRecoveryExportCreated> {
|
||||
let (created, _) = load_recovery_export_decoded(api, export_id).await?;
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
async fn load_recovery_export_decoded(
|
||||
api: Arc<ECStore>,
|
||||
export_id: &str,
|
||||
) -> Result<(IlmRecoveryExportCreated, IlmRecoveryExport)> {
|
||||
let object = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, export_id)?;
|
||||
let encoded = config_boundary::read_config_limited_preserve_empty(api, &object, MAX_ILM_RECOVERY_EXPORT_SIZE).await?;
|
||||
let export = IlmRecoveryExport::decode(export_id, &encoded)?;
|
||||
let content_sha256 = hex_sha256(&encoded, ToOwned::to_owned);
|
||||
Ok((
|
||||
IlmRecoveryExportCreated {
|
||||
export_id: export.export_id.clone(),
|
||||
content_sha256,
|
||||
encoded,
|
||||
replayed: false,
|
||||
},
|
||||
export,
|
||||
))
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportCreated {
|
||||
fn with_replayed(mut self) -> Self {
|
||||
self.replayed = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_exportable_control(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(api, control_id, &ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_no_lock(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(
|
||||
api,
|
||||
control_id,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_with_options(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
options: &ObjectOptions,
|
||||
) -> Result<ObservedIlmRecoveryControl> {
|
||||
let object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, control_id).map_err(Error::other)?;
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_limited_preserve_empty_with_metadata(api, &object, options, MAX_ILM_RECOVERY_CONTROL_SIZE)
|
||||
.await?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery control is missing an ETag"))?;
|
||||
let control = IlmRecoveryControl::decode(control_id, &data).map_err(Error::other)?;
|
||||
if control.identity.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| control.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&control.observed_source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery control is not exportable"));
|
||||
}
|
||||
Ok(ObservedIlmRecoveryControl { control, etag })
|
||||
}
|
||||
|
||||
async fn current_observation_under_proof_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
expected: &IlmRecoveryExportObservation,
|
||||
proof: &crate::services::notification_sys::IlmRecoveryExportFleetProofToken,
|
||||
) -> Result<(IlmRecoveryExportObservation, Vec<u8>)> {
|
||||
let observed_control = load_exportable_control_no_lock(api.clone(), &expected.control_id).await?;
|
||||
let observed_source = observe_export_source_no_lock(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
let source_bytes = observed_source
|
||||
.canonical_data
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copies diverge"))?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(proof).await
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok((
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: expected.control_id.clone(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(proof),
|
||||
},
|
||||
source_bytes,
|
||||
))
|
||||
}
|
||||
|
||||
async fn observe_export_source(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
if canonical_path.is_empty()
|
||||
|| canonical_path.starts_with('/')
|
||||
|| canonical_path.ends_with('/')
|
||||
|| canonical_path.split('/').any(str::is_empty)
|
||||
|| !is_legacy_export_schema(source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source identity is invalid"));
|
||||
}
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, canonical_path).await?;
|
||||
let _guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
observe_export_source_no_lock(api, canonical_path, source_schema).await
|
||||
}
|
||||
|
||||
async fn observe_export_source_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
let mut copies = Vec::new();
|
||||
let mut canonical: Option<(String, String, Vec<u8>)> = None;
|
||||
let mut consistent = true;
|
||||
for set in api.all_set_disks() {
|
||||
let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index);
|
||||
let result = config_boundary::read_config_limited_preserve_empty_with_metadata(
|
||||
set,
|
||||
canonical_path,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
MAX_LEGACY_TIER_DELETE_SOURCE_SIZE,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok((data, metadata)) => {
|
||||
if data.is_empty() || data.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE {
|
||||
return Err(Error::other("ILM recovery export source exceeds its protocol size limit"));
|
||||
}
|
||||
validate_legacy_tier_delete_recovery_source(canonical_path, source_schema, &data)?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copy is missing an ETag"))?;
|
||||
let content_sha256 = hex_sha256(&data, ToOwned::to_owned);
|
||||
let encoded_len =
|
||||
u64::try_from(data.len()).map_err(|_| Error::other("ILM recovery export source length does not fit u64"))?;
|
||||
copies.push(IlmRecoverySourceCopy {
|
||||
authority,
|
||||
canonical_path: canonical_path.to_string(),
|
||||
etag: etag.clone(),
|
||||
encoded_len,
|
||||
content_sha256: content_sha256.clone(),
|
||||
});
|
||||
match canonical.as_ref() {
|
||||
Some((first_etag, first_digest, first_data)) => {
|
||||
consistent &= first_etag == &etag && first_digest == &content_sha256 && first_data == &data;
|
||||
}
|
||||
None => canonical = Some((etag, content_sha256, data)),
|
||||
}
|
||||
}
|
||||
Err(err) if export_source_is_missing(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let Some((source_etag, content_sha256, source_bytes)) = canonical else {
|
||||
return Err(Error::ConfigNotFound);
|
||||
};
|
||||
let generation =
|
||||
IlmRecoverySourceGeneration::new(source_schema, source_etag, content_sha256, copies).map_err(Error::other)?;
|
||||
Ok(ObservedIlmRecoverySource {
|
||||
generation,
|
||||
canonical_data: consistent.then_some(source_bytes),
|
||||
})
|
||||
}
|
||||
|
||||
fn export_source_is_missing(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
)
|
||||
}
|
||||
|
||||
fn build_export_from_source(
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
created_at_unix_nanos: i64,
|
||||
source_bytes_base64: &str,
|
||||
) -> Result<IlmRecoveryExport> {
|
||||
let retain_until_unix_nanos = created_at_unix_nanos
|
||||
.checked_add(EXPORT_RETENTION_NANOS)
|
||||
.ok_or_else(|| Error::other("ILM recovery export retention timestamp overflow"))?;
|
||||
let export = IlmRecoveryExport {
|
||||
export_id: recovery_export_id(&observation.control_id, &observation.source_generation)?,
|
||||
control_id: observation.control_id.clone(),
|
||||
protocol: observation.protocol,
|
||||
control_etag: observation.control_etag.clone(),
|
||||
control_revision: observation.control_revision,
|
||||
classification: observation.classification,
|
||||
canonical_source_path: observation.canonical_source_path.clone(),
|
||||
source_generation: observation.source_generation.clone(),
|
||||
topology_generation: observation.topology_generation.clone(),
|
||||
member_epochs_sha256: observation.member_epochs_sha256.clone(),
|
||||
creator_sha256: creator_sha256.to_string(),
|
||||
created_at_unix_nanos,
|
||||
retain_until_unix_nanos,
|
||||
source_bytes_base64: source_bytes_base64.to_string(),
|
||||
};
|
||||
export.validate()?;
|
||||
Ok(export)
|
||||
}
|
||||
|
||||
pub(crate) fn recovery_export_id(control_id: &str, generation: &IlmRecoverySourceGeneration) -> Result<String> {
|
||||
validate_sha256(control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&generation.content_sha256, "ILM recovery export source checksum is invalid")?;
|
||||
validate_sha256(&generation.copy_set_sha256, "ILM recovery export copy-set checksum is invalid")?;
|
||||
let mut data = Vec::new();
|
||||
for part in [control_id, &generation.content_sha256, &generation.copy_set_sha256] {
|
||||
data.extend_from_slice(&(part.len() as u64).to_be_bytes());
|
||||
data.extend_from_slice(part.as_bytes());
|
||||
}
|
||||
Ok(hex_sha256(&data, ToOwned::to_owned))
|
||||
}
|
||||
|
||||
fn export_matches_observation(export: &IlmRecoveryExport, observation: &IlmRecoveryExportObservation) -> bool {
|
||||
export.control_id == observation.control_id
|
||||
&& export.protocol == observation.protocol
|
||||
&& export.classification == observation.classification
|
||||
&& export.canonical_source_path == observation.canonical_source_path
|
||||
&& export.source_generation == observation.source_generation
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct IlmRecoveryExportInventory {
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
creations: Vec<(i64, String)>,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportInventory {
|
||||
fn check(&self, creator_sha256: &str, candidate_len: usize, now: i64) -> Result<()> {
|
||||
let recent_after = now.saturating_sub(60 * 1_000_000_000);
|
||||
let cluster_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, _)| *created_at > recent_after)
|
||||
.count();
|
||||
let actor_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, creator)| *created_at > recent_after && creator == creator_sha256)
|
||||
.count();
|
||||
check_export_admission(self.count, self.bytes, actor_recent, cluster_recent, candidate_len)
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_export_inventory(api: Arc<ECStore>) -> Result<IlmRecoveryExportInventory> {
|
||||
let mut marker = None;
|
||||
let mut seen_markers = HashSet::new();
|
||||
let mut inventory = IlmRecoveryExportInventory::default();
|
||||
loop {
|
||||
let page = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("{ILM_RECOVERY_EXPORT_PREFIX}/"),
|
||||
marker.clone(),
|
||||
None,
|
||||
1_000,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
for object in page.objects {
|
||||
let (_, export_id) = recovery_export_id_from_record_object_name(&object.name)?;
|
||||
let (stored, export) = load_recovery_export_decoded(api.clone(), &export_id).await?;
|
||||
inventory.count = inventory
|
||||
.count
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| Error::other("ILM recovery export count overflow"))?;
|
||||
inventory.bytes = inventory
|
||||
.bytes
|
||||
.checked_add(u64::try_from(stored.encoded.len()).map_err(|_| Error::other("ILM recovery export size overflow"))?)
|
||||
.ok_or_else(|| Error::other("ILM recovery export byte total overflow"))?;
|
||||
inventory
|
||||
.creations
|
||||
.push((export.created_at_unix_nanos, export.creator_sha256));
|
||||
}
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
let next = page
|
||||
.next_continuation_token
|
||||
.ok_or_else(|| Error::other("ILM recovery export inventory omitted its continuation marker"))?;
|
||||
marker = Some(record_export_inventory_marker(&mut seen_markers, next)?);
|
||||
}
|
||||
Ok(inventory)
|
||||
}
|
||||
|
||||
fn record_export_inventory_marker(seen_markers: &mut HashSet<String>, next: String) -> Result<String> {
|
||||
if !seen_markers.insert(next.clone()) {
|
||||
return Err(Error::other("ILM recovery export inventory repeated its continuation marker"));
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
fn check_export_admission(
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
actor_recent: usize,
|
||||
cluster_recent: usize,
|
||||
candidate_len: usize,
|
||||
) -> Result<()> {
|
||||
let candidate_len = u64::try_from(candidate_len).map_err(|_| Error::other("ILM recovery export size does not fit u64"))?;
|
||||
if count >= MAX_ILM_RECOVERY_EXPORTS
|
||||
|| bytes
|
||||
.checked_add(candidate_len)
|
||||
.is_none_or(|total| total > MAX_ILM_RECOVERY_EXPORT_BYTES)
|
||||
|| actor_recent >= MAX_ACTOR_EXPORTS_PER_MINUTE
|
||||
|| cluster_recent >= MAX_CLUSTER_EXPORTS_PER_MINUTE
|
||||
{
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_legacy_export_schema(schema: &str) -> bool {
|
||||
matches!(schema, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA | TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA)
|
||||
}
|
||||
|
||||
fn validate_sha256(value: &str, message: &'static str) -> Result<()> {
|
||||
if !is_sha256_checksum(value)
|
||||
|| value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(Error::other(message));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn now_unix_nanos() -> Result<i64> {
|
||||
i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.map_err(|_| Error::other("ILM recovery export timestamp does not fit i64"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy;
|
||||
|
||||
const PINNED_V1_EXPORT: &[u8] = br#"{"schema":"rustfs-ilm-recovery-export-v1","content_sha256":"3dfb3ec3892256e909de1211c1a963ca7008963ff32b3a869f7161a7b9b44028","export":{"export_id":"2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105","control_id":"0fcd568a5cb9bdb4677b69354b11ee415af8f784519cff3da49a26f84eaee7f2","protocol":"tier_delete_journal","control_etag":"control-etag","control_revision":1,"classification":"retained_ambiguous","canonical_source_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","source_generation":{"source_schema":"rustfs-tier-delete-journal-v1","source_etag":"etag-a","content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd","copy_set_sha256":"5a7406115b6c3923ffe79dcd1f43ccae7beed786e557163f019dd10ec409a653","copies":[{"authority":"pool-0/set-0","canonical_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","etag":"etag-a","encoded_len":81,"content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd"}]},"topology_generation":"e6e2b826e31fca5c36125c48f130dcb6f961e698ff8a8776a1f290cf0892e8e6","member_epochs_sha256":"612dd8a861161819a4ad8f6f3e2a0567602877c043a2353ca933a13c78dc0ed4","creator_sha256":"50c9c4aeb40b5b206b6d98f516f8b8c0efd29ce2e56a76b345fb9240c225a1b7","created_at_unix_nanos":1000000000,"retain_until_unix_nanos":7776001000000000,"source_bytes_base64":"eyJ2ZXJzaW9uIjoxLCJvYmpfbmFtZSI6ImxlZ2FjeS9yZW1vdGUiLCJ2ZXJzaW9uX2lkIjoib3BhcXVlIiwidGllcl9uYW1lIjoiV0FSTSJ9"}}"#;
|
||||
|
||||
fn legacy_source() -> Vec<u8> {
|
||||
br#"{"version":1,"obj_name":"legacy/remote","version_id":"opaque","tier_name":"WARM"}"#.to_vec()
|
||||
}
|
||||
|
||||
fn observation() -> IlmRecoveryExportObservation {
|
||||
let source = legacy_source();
|
||||
let source_path = super::super::tier_delete_journal::tier_delete_journal_object_name(
|
||||
&super::super::tier_delete_journal::decode_tier_delete_journal_entry(&source).expect("legacy fixture should decode"),
|
||||
);
|
||||
let source_sha256 = hex_sha256(&source, ToOwned::to_owned);
|
||||
let generation = IlmRecoverySourceGeneration::new(
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA,
|
||||
"etag-a",
|
||||
source_sha256.clone(),
|
||||
vec![IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.clone(),
|
||||
etag: "etag-a".to_string(),
|
||||
encoded_len: source.len() as u64,
|
||||
content_sha256: source_sha256,
|
||||
}],
|
||||
)
|
||||
.expect("generation should be valid");
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: hex_sha256(b"control", ToOwned::to_owned),
|
||||
protocol: IlmRecoveryProtocol::TierDeleteJournal,
|
||||
control_etag: "control-etag".to_string(),
|
||||
control_revision: 1,
|
||||
classification: IlmRecoveryClassification::RetainedAmbiguous,
|
||||
canonical_source_path: source_path,
|
||||
source_generation: generation,
|
||||
topology_generation: hex_sha256(b"topology", ToOwned::to_owned),
|
||||
member_epochs_sha256: hex_sha256(b"epochs", ToOwned::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_round_trip_is_strict_and_deterministic() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.expect("export should be valid");
|
||||
assert_eq!(
|
||||
export.export_id,
|
||||
recovery_export_id(&observed.control_id, &observed.source_generation).unwrap()
|
||||
);
|
||||
let encoded = export.encode().expect("export should encode");
|
||||
assert_eq!(encoded, PINNED_V1_EXPORT, "v1 export wire format must remain pinned");
|
||||
assert_eq!(IlmRecoveryExport::decode(&export.export_id, &encoded).unwrap(), export);
|
||||
assert_eq!(
|
||||
IlmRecoveryExport::decode("2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105", PINNED_V1_EXPORT)
|
||||
.unwrap(),
|
||||
export,
|
||||
);
|
||||
|
||||
let path = recovery_export_record_object_name(export.protocol, &export.export_id).unwrap();
|
||||
let durable = super::super::durable_namespace::validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("export should be registered as a durable ILM record");
|
||||
assert_eq!(durable.namespace, "recovery-export");
|
||||
assert_eq!(durable.id_kind, "export_id");
|
||||
assert_eq!(durable.id, export.export_id);
|
||||
|
||||
let mut wrong_source = export.clone();
|
||||
wrong_source.source_bytes_base64 = base64_simd::STANDARD.encode_to_string(b"changed");
|
||||
assert!(wrong_source.encode().is_err());
|
||||
|
||||
let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
|
||||
persisted["unknown"] = serde_json::json!(true);
|
||||
assert!(IlmRecoveryExport::decode(&export.export_id, &serde_json::to_vec(&persisted).unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_inventory_rejects_non_adjacent_continuation_cycles() {
|
||||
let mut seen = HashSet::new();
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "a".to_string()).unwrap(), "a");
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "b".to_string()).unwrap(), "b");
|
||||
record_export_inventory_marker(&mut seen, "a".to_string())
|
||||
.expect_err("a non-adjacent continuation marker cycle must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_path_rejects_noncanonical_shards() {
|
||||
let id = hex_sha256(b"export", ToOwned::to_owned);
|
||||
let path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &id).unwrap();
|
||||
assert_eq!(recovery_export_id_from_record_object_name(&path).unwrap().1, id);
|
||||
let wrong_shard = path.replacen(&format!("/{}/", &id[..2]), "/zz/", 1);
|
||||
assert!(recovery_export_id_from_record_object_name(&wrong_shard).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_replay_survives_fleet_rotation_but_not_source_change() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut rotated = observed;
|
||||
rotated.control_etag = "new-control-etag".to_string();
|
||||
rotated.control_revision += 1;
|
||||
rotated.topology_generation = hex_sha256(b"new-topology", ToOwned::to_owned);
|
||||
rotated.member_epochs_sha256 = hex_sha256(b"new-members", ToOwned::to_owned);
|
||||
assert!(export_matches_observation(&export, &rotated));
|
||||
|
||||
rotated.source_generation.content_sha256 = hex_sha256(b"changed", ToOwned::to_owned);
|
||||
assert!(!export_matches_observation(&export, &rotated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_admission_enforces_exact_count_byte_and_rate_boundaries() {
|
||||
assert!(check_export_admission(9_999, MAX_ILM_RECOVERY_EXPORT_BYTES - 1, 9, 99, 1).is_ok());
|
||||
assert!(check_export_admission(10_000, 0, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, MAX_ILM_RECOVERY_EXPORT_BYTES, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 10, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 0, 100, 1).is_err());
|
||||
}
|
||||
}
|
||||
@@ -82,8 +82,8 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
|
||||
const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
|
||||
@@ -884,6 +884,23 @@ struct PersistedTierDeleteJournalEntry {
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn validate_legacy_recovery_shape(&self) -> Result<()> {
|
||||
let has_later_version_fields = self.version_id_exact.is_some()
|
||||
|| self.version_state.is_some()
|
||||
|| self.state.is_some()
|
||||
|| self.source.is_some()
|
||||
|| self.dispatch.is_some();
|
||||
match self.version {
|
||||
1 if self.backend_identity.is_none() && !has_later_version_fields => Ok(()),
|
||||
TIER_DELETE_JOURNAL_VERSION if self.backend_identity.is_some() && !has_later_version_fields => Ok(()),
|
||||
1 => Err(Error::other("tier delete journal v1 entry contains fields from a later version")),
|
||||
TIER_DELETE_JOURNAL_VERSION => Err(Error::other(
|
||||
"tier delete journal v2 entry is missing its identity or contains fields from a later version",
|
||||
)),
|
||||
_ => Err(Error::other("tier delete journal is not an exportable legacy version")),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
@@ -5531,6 +5548,12 @@ fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&s
|
||||
.then_some(identity)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_path(object_name: &str) -> Result<()> {
|
||||
canonical_legacy_tier_delete_journal_identity(object_name)
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| Error::other("legacy tier delete journal path is not canonical"))
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
|
||||
match entry.persisted_version {
|
||||
1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
|
||||
@@ -5539,6 +5562,21 @@ fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static st
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_source(object_name: &str, source_schema: &str, data: &[u8]) -> Result<()> {
|
||||
validate_legacy_tier_delete_recovery_path(object_name)?;
|
||||
let persisted: PersistedTierDeleteJournalEntry =
|
||||
serde_json::from_slice(data).map_err(|err| Error::other_with_context("decode tier delete journal failed", err))?;
|
||||
persisted.validate_legacy_recovery_shape()?;
|
||||
let entry = persisted.into_jentry()?;
|
||||
let Some((decoded_schema, _)) = legacy_tier_delete_recovery_descriptor(&entry) else {
|
||||
return Err(Error::other("tier delete journal is not an exportable legacy version"));
|
||||
};
|
||||
if decoded_schema != source_schema || tier_delete_journal_object_name(&entry) != object_name {
|
||||
return Err(Error::other("legacy tier delete journal identity does not match its recovery source"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_matches(
|
||||
control: &IlmRecoveryControl,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
@@ -6140,17 +6178,18 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION,
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent,
|
||||
TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
PersistedTierDeleteJournalEntry, TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE,
|
||||
TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX,
|
||||
TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION,
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TIER_DELETE_JOURNAL_VERSION, TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState,
|
||||
TierDeleteDispatchParent, TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest,
|
||||
encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity,
|
||||
tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name,
|
||||
tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name,
|
||||
tier_delete_source_matches_dispatch_scope,
|
||||
tier_delete_source_matches_dispatch_scope, validate_legacy_tier_delete_recovery_source,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -6609,6 +6648,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_recovery_export_rejects_fields_from_later_journal_versions() {
|
||||
let later = bound_v6_journal_entry(TierDeleteJournalState::Prepared);
|
||||
let v1 = PersistedTierDeleteJournalEntry {
|
||||
version: 1,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "opaque".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: None,
|
||||
version_id_exact: None,
|
||||
version_state: None,
|
||||
state: None,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
let mut v2 = v1.clone();
|
||||
v2.version = TIER_DELETE_JOURNAL_VERSION;
|
||||
v2.backend_identity = Some([7; 32]);
|
||||
|
||||
let assert_rejected = |persisted: PersistedTierDeleteJournalEntry, schema: &str| {
|
||||
let normalized = persisted
|
||||
.clone()
|
||||
.into_jentry()
|
||||
.expect("the generic compatibility decoder should demonstrate the discarded field");
|
||||
let object_name = tier_delete_journal_object_name(&normalized);
|
||||
let encoded = serde_json::to_vec(&persisted).expect("mixed-version journal fixture should encode");
|
||||
let err = validate_legacy_tier_delete_recovery_source(&object_name, schema, &encoded)
|
||||
.expect_err("legacy recovery export must reject fields from later versions");
|
||||
assert!(err.to_string().contains("later version"));
|
||||
};
|
||||
|
||||
let mut invalid_v1 = Vec::new();
|
||||
let mut with_backend = v1.clone();
|
||||
with_backend.backend_identity = Some([7; 32]);
|
||||
invalid_v1.push(with_backend);
|
||||
for persisted in [&v1, &v2] {
|
||||
let schema = if persisted.version == 1 {
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA
|
||||
} else {
|
||||
TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA
|
||||
};
|
||||
let mut invalid = Vec::new();
|
||||
let mut with_exact = persisted.clone();
|
||||
with_exact.version_id_exact = Some(false);
|
||||
invalid.push(with_exact);
|
||||
let mut with_version_state = persisted.clone();
|
||||
with_version_state.version_state = Some(rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
invalid.push(with_version_state);
|
||||
let mut with_state = persisted.clone();
|
||||
with_state.state = Some(TierDeleteJournalState::Committed);
|
||||
invalid.push(with_state);
|
||||
let mut with_source = persisted.clone();
|
||||
with_source.source = later.source.clone();
|
||||
invalid.push(with_source);
|
||||
let mut with_dispatch = persisted.clone();
|
||||
with_dispatch.dispatch = later.dispatch.clone();
|
||||
invalid.push(with_dispatch);
|
||||
for record in invalid {
|
||||
assert_rejected(record, schema);
|
||||
}
|
||||
}
|
||||
for record in invalid_v1 {
|
||||
assert_rejected(record, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::bucket::lifecycle::tier_sweeper::{
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -273,6 +273,14 @@ pub struct TransitionTransactionInit {
|
||||
|
||||
impl TransitionTransaction {
|
||||
pub fn new(init: TransitionTransactionInit) -> Result<Self> {
|
||||
Self::new_with_initial_state(init, TransitionTransactionState::UploadStarted)
|
||||
}
|
||||
|
||||
pub(crate) fn new_compact(init: TransitionTransactionInit) -> Result<Self> {
|
||||
Self::new_with_initial_state(init, TransitionTransactionState::UploadOutcomeUnknown)
|
||||
}
|
||||
|
||||
fn new_with_initial_state(init: TransitionTransactionInit, state: TransitionTransactionState) -> Result<Self> {
|
||||
let remote_object =
|
||||
canonical_transition_remote_object(init.deployment_id, &init.source.bucket, init.transaction_id, init.write_id)?;
|
||||
let transaction = Self {
|
||||
@@ -286,7 +294,7 @@ impl TransitionTransaction {
|
||||
backend_fingerprint: init.backend_fingerprint,
|
||||
remote_object,
|
||||
remote_version: TransitionRemoteVersion::unknown(),
|
||||
state: TransitionTransactionState::UploadStarted,
|
||||
state,
|
||||
not_after_unix_nanos: init.not_after_unix_nanos,
|
||||
};
|
||||
transaction.validate()?;
|
||||
@@ -356,7 +364,7 @@ impl TransitionTransaction {
|
||||
remote_version: Option<TransitionRemoteVersion>,
|
||||
) -> Result<TransitionTransactionFence> {
|
||||
self.check_fence(fence)?;
|
||||
if !state_change_allowed(self.state, next) {
|
||||
if !state_change_allowed_at(self.state, next, self.revision) {
|
||||
return Err(TransitionTransactionError::InvalidStateChange {
|
||||
from: self.state,
|
||||
to: next,
|
||||
@@ -388,6 +396,14 @@ impl TransitionTransaction {
|
||||
}
|
||||
self.remote_version = TransitionRemoteVersion::unknown();
|
||||
}
|
||||
TransitionTransactionState::LocalCommitStarted if self.state == TransitionTransactionState::UploadOutcomeUnknown => {
|
||||
let remote_version =
|
||||
remote_version.ok_or(TransitionTransactionError::Corrupt("compact local commit requires remote version"))?;
|
||||
if remote_version.is_unknown() {
|
||||
return Err(TransitionTransactionError::Corrupt("compact local commit requires known remote version"));
|
||||
}
|
||||
self.remote_version = remote_version;
|
||||
}
|
||||
TransitionTransactionState::LocalCommitStarted | TransitionTransactionState::Committed => {
|
||||
if let Some(remote_version) = remote_version
|
||||
&& remote_version != self.remote_version
|
||||
@@ -640,7 +656,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
|
||||
) -> EcstoreResult<()> {
|
||||
let object = transition_transaction_record_object_name(next.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let revision_is_next = expected.revision.checked_add(1) == Some(next.revision);
|
||||
let state_is_next = state_change_allowed(expected.state, next.state)
|
||||
let state_is_next = state_change_allowed_at(expected.state, next.state, expected.revision)
|
||||
|| matches!(
|
||||
(expected.state, next.state),
|
||||
(
|
||||
@@ -954,6 +970,10 @@ pub enum TransitionOperatorError {
|
||||
expected: String,
|
||||
actual: TransitionOperatorProbe,
|
||||
},
|
||||
#[error("transition recovery control is stale")]
|
||||
StaleRecoveryControl,
|
||||
#[error("transition recovery control is not eligible for operator retry")]
|
||||
RetryNotAllowed,
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
@@ -962,6 +982,179 @@ pub enum TransitionOperatorError {
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionRecoveryRetryStatus {
|
||||
pub control_id: String,
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub control_revision: u64,
|
||||
pub attempt_count: u64,
|
||||
pub consecutive_failure_count: u32,
|
||||
pub last_error_code: IlmRecoveryErrorCode,
|
||||
pub source_generation_sha256: String,
|
||||
pub copy_set_sha256: String,
|
||||
pub retry_ready: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub retry_not_ready_reason: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionRecoveryRetryResult {
|
||||
pub control_id: String,
|
||||
pub transaction_id: Uuid,
|
||||
pub previous_revision: u64,
|
||||
pub revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub attempt_count: u64,
|
||||
pub source_generation_sha256: String,
|
||||
}
|
||||
|
||||
struct TransitionRecoveryRetryContext {
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
transaction: TransitionTransaction,
|
||||
source_generation_sha256: String,
|
||||
}
|
||||
|
||||
fn transition_recovery_retry_readiness(control: &IlmRecoveryControl) -> (bool, Option<&'static str>) {
|
||||
if control.owner.is_some() {
|
||||
return (false, Some("attempt_owned"));
|
||||
}
|
||||
match control.classification {
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired => (true, None),
|
||||
IlmRecoveryClassification::Retrying => (false, Some("already_retrying")),
|
||||
IlmRecoveryClassification::Corrupt => (false, Some("source_corrupt")),
|
||||
IlmRecoveryClassification::Abandoned => (false, Some("source_abandoned")),
|
||||
IlmRecoveryClassification::Terminal => (false, Some("source_terminal")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_transition_recovery_retry_context(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryContext> {
|
||||
let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(observed) => observed,
|
||||
Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
let transaction_id = Uuid::parse_str(&observed.control.identity.stable_operation_identity)
|
||||
.ok()
|
||||
.filter(|transaction_id| !transaction_id.is_nil())
|
||||
.ok_or(TransitionOperatorError::StaleRecoveryControl)?;
|
||||
let canonical_path = transition_transaction_record_object_name(transaction_id)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if observed.control.identity.canonical_source_path != canonical_path
|
||||
|| observed.control.identity.record_class != "transition_transaction_v1"
|
||||
{
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
|
||||
Ok(transaction) => transaction,
|
||||
Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
let source = observe_recovery_source(api, &canonical_path, TRANSITION_TRANSACTION_SCHEMA)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source.generation == observed.control.observed_source_generation
|
||||
&& source
|
||||
.canonical_data
|
||||
.as_deref()
|
||||
.is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
|
||||
if !exact_source {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let generation = serde_json::to_vec(&observed.control.observed_source_generation)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
Ok(TransitionRecoveryRetryContext {
|
||||
observed,
|
||||
transaction,
|
||||
source_generation_sha256: hex_sha256(&generation, ToOwned::to_owned),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_recovery_retry_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryStatus> {
|
||||
let context = load_transition_recovery_retry_context(api, control_id).await?;
|
||||
let (retry_ready, retry_not_ready_reason) = transition_recovery_retry_readiness(&context.observed.control);
|
||||
Ok(TransitionRecoveryRetryStatus {
|
||||
control_id: control_id.to_string(),
|
||||
transaction_id: context.transaction.transaction_id,
|
||||
state: context.transaction.state,
|
||||
classification: context.observed.control.classification,
|
||||
control_revision: context.observed.control.revision,
|
||||
attempt_count: context.observed.control.attempt_count,
|
||||
consecutive_failure_count: context.observed.control.consecutive_failure_count,
|
||||
last_error_code: context.observed.control.last_error_code,
|
||||
source_generation_sha256: context.source_generation_sha256,
|
||||
copy_set_sha256: context.observed.control.observed_source_generation.copy_set_sha256.clone(),
|
||||
retry_ready,
|
||||
retry_not_ready_reason,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn retry_transition_recovery_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
expected_control_revision: u64,
|
||||
expected_source_generation_sha256: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryResult> {
|
||||
let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, control_id)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
let retry_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_object}.recovery-lock"))
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
let retry_guard = retry_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
let context = load_transition_recovery_retry_context(api.clone(), control_id).await?;
|
||||
let (retry_ready, _) = transition_recovery_retry_readiness(&context.observed.control);
|
||||
if !retry_ready {
|
||||
return Err(TransitionOperatorError::RetryNotAllowed);
|
||||
}
|
||||
if retry_guard.is_lock_lost()
|
||||
|| expected_control_revision == 0
|
||||
|| context.observed.control.revision != expected_control_revision
|
||||
|| context.source_generation_sha256 != expected_source_generation_sha256
|
||||
{
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let previous_revision = context.observed.control.revision;
|
||||
let mut next = context.observed.control.clone();
|
||||
next.retry_for_operator(&context.observed.control.observed_source_generation)
|
||||
.map_err(|_| TransitionOperatorError::RetryNotAllowed)?;
|
||||
if retry_guard.is_lock_lost() {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
save_recovery_control_if_current(api.clone(), &context.observed, &next)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
Error::PreconditionFailed => TransitionOperatorError::StaleRecoveryControl,
|
||||
err => TransitionOperatorError::Store(err),
|
||||
})?;
|
||||
let persisted = load_recovery_control(api, IlmRecoveryProtocol::TransitionTransaction, control_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
if retry_guard.is_lock_lost() || persisted.control != next {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
Ok(TransitionRecoveryRetryResult {
|
||||
control_id: control_id.to_string(),
|
||||
transaction_id: context.transaction.transaction_id,
|
||||
previous_revision,
|
||||
revision: persisted.control.revision,
|
||||
classification: persisted.control.classification,
|
||||
attempt_count: persisted.control.attempt_count,
|
||||
source_generation_sha256: context.source_generation_sha256,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
@@ -1706,12 +1899,27 @@ async fn local_commit_matches_transaction(api: Arc<ECStore>, transaction: &Trans
|
||||
.get_object_info(&transaction.source.bucket, &transaction.source.object, &opts)
|
||||
.await?;
|
||||
let transitioned = &object.transitioned_object;
|
||||
Ok(transitioned.status == TRANSITION_COMPLETE
|
||||
Ok(local_object_matches_transition_source(&object, &transaction.source)
|
||||
&& transitioned.status == TRANSITION_COMPLETE
|
||||
&& transitioned.name == transaction.remote_object
|
||||
&& transitioned.tier == transaction.tier_name
|
||||
&& transitioned.version_id == transaction.remote_version.tier_delete_version_id().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn local_object_matches_transition_source(object: &ObjectInfo, source: &TransitionSourceIdentity) -> bool {
|
||||
let observed_version_id = object.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let observed_mod_time = object
|
||||
.mod_time
|
||||
.and_then(|mod_time| i64::try_from(mod_time.unix_timestamp_nanos()).ok());
|
||||
object.bucket == source.bucket
|
||||
&& object.name == source.object
|
||||
&& observed_version_id == source.version_id
|
||||
&& object.data_dir == Some(source.data_dir)
|
||||
&& observed_mod_time == Some(source.mod_time_unix_nanos)
|
||||
&& object.size == source.size
|
||||
&& object.etag.as_deref() == Some(source.etag.as_str())
|
||||
}
|
||||
|
||||
fn transition_source_lookup_options(transaction: &TransitionTransaction) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
version_id: match transaction.source.version_mode {
|
||||
@@ -1954,7 +2162,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn state_change_allowed(from: TransitionTransactionState, to: TransitionTransactionState) -> bool {
|
||||
fn state_change_allowed_at(from: TransitionTransactionState, to: TransitionTransactionState, revision: u64) -> bool {
|
||||
matches!(
|
||||
(from, to),
|
||||
(TransitionTransactionState::UploadStarted, TransitionTransactionState::Uploaded)
|
||||
@@ -1966,7 +2174,9 @@ fn state_change_allowed(from: TransitionTransactionState, to: TransitionTransact
|
||||
| (TransitionTransactionState::UploadOutcomeUnknown, TransitionTransactionState::Uploaded)
|
||||
| (TransitionTransactionState::Uploaded, TransitionTransactionState::LocalCommitStarted)
|
||||
| (TransitionTransactionState::LocalCommitStarted, TransitionTransactionState::Committed)
|
||||
)
|
||||
) || (revision == 1
|
||||
&& from == TransitionTransactionState::UploadOutcomeUnknown
|
||||
&& to == TransitionTransactionState::LocalCommitStarted)
|
||||
}
|
||||
|
||||
fn state_requires_known_remote_version(state: TransitionTransactionState) -> bool {
|
||||
@@ -2183,6 +2393,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_commit_proof_requires_the_complete_source_identity() {
|
||||
let source = source_identity(TransitionSourceVersionMode::Versioned);
|
||||
let exact = ObjectInfo {
|
||||
bucket: source.bucket.clone(),
|
||||
name: source.object.clone(),
|
||||
version_id: source.version_id,
|
||||
data_dir: Some(source.data_dir),
|
||||
mod_time: Some(
|
||||
time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(source.mod_time_unix_nanos))
|
||||
.expect("source timestamp should be valid"),
|
||||
),
|
||||
size: source.size,
|
||||
etag: Some(source.etag.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(local_object_matches_transition_source(&exact, &source));
|
||||
|
||||
let mut changed = exact.clone();
|
||||
changed.version_id = Some(Uuid::new_v4());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.data_dir = Some(Uuid::new_v4());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.mod_time = changed.mod_time.map(|value| value + Duration::from_nanos(1));
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.size += 1;
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact;
|
||||
changed.etag = Some("different-etag".to_string());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
@@ -2384,6 +2629,57 @@ mod tests {
|
||||
assert_eq!(transaction.state, TransitionTransactionState::Committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_state_sequence_is_distinguishable_and_keeps_legacy_edges_strict() {
|
||||
let init = TransitionTransactionInit {
|
||||
deployment_id: Uuid::new_v4(),
|
||||
transaction_id: Uuid::new_v4(),
|
||||
owner_epoch: Uuid::new_v4(),
|
||||
write_id: Uuid::new_v4(),
|
||||
source: source_identity(TransitionSourceVersionMode::Versioned),
|
||||
tier_name: "warm-tier".to_string(),
|
||||
backend_fingerprint: BACKEND_FINGERPRINT,
|
||||
not_after_unix_nanos: 1_780_000_000_000_000_000,
|
||||
};
|
||||
let mut compact = TransitionTransaction::new_compact(init).expect("compact transaction should be created");
|
||||
assert_eq!(compact.state, TransitionTransactionState::UploadOutcomeUnknown);
|
||||
assert_eq!(compact.revision, 1);
|
||||
let remote_version = TransitionRemoteVersion::versioned(Uuid::new_v4().to_string());
|
||||
let fence = compact
|
||||
.advance(
|
||||
compact.fence(),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
Some(remote_version.clone()),
|
||||
)
|
||||
.expect("compact upload should persist its exact candidate at the local commit fence");
|
||||
assert_eq!(fence.revision, 2);
|
||||
assert_eq!(compact.remote_version, remote_version);
|
||||
assert_eq!(compact.state, TransitionTransactionState::LocalCommitStarted);
|
||||
let encoded = compact
|
||||
.encode()
|
||||
.expect("compact transaction should encode as v1-compatible bytes");
|
||||
assert_eq!(
|
||||
TransitionTransaction::decode(compact.transaction_id, &encoded).expect("compact transaction should decode"),
|
||||
compact
|
||||
);
|
||||
|
||||
let mut legacy_unknown = new_transaction();
|
||||
legacy_unknown
|
||||
.advance(legacy_unknown.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("legacy transaction should persist its pre-upload fence");
|
||||
assert!(matches!(
|
||||
legacy_unknown.advance(
|
||||
legacy_unknown.fence(),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
Some(TransitionRemoteVersion::unversioned()),
|
||||
),
|
||||
Err(TransitionTransactionError::InvalidStateChange {
|
||||
from: TransitionTransactionState::UploadOutcomeUnknown,
|
||||
to: TransitionTransactionState::LocalCommitStarted,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_pending_requires_exact_proof_and_state_specific_decision() {
|
||||
let mut transaction = new_transaction();
|
||||
|
||||
@@ -66,6 +66,7 @@ pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
pub(crate) use replication_object_decision_boundary::replication_etags_match;
|
||||
pub use replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
@@ -88,5 +89,6 @@ pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub use replication_target_boundary::VersionIdentityCapability;
|
||||
pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_filemeta::ObjectPartInfo;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::ReplicationMultipartPlanError;
|
||||
pub use rustfs_replication::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
|
||||
@@ -56,6 +56,8 @@ use super::replication_storage_boundary::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
|
||||
#[cfg(test)]
|
||||
use super::replication_target_boundary::VersionIdentityCapability;
|
||||
use super::replication_target_boundary::{
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
|
||||
@@ -63,7 +65,7 @@ use super::replication_target_boundary::{
|
||||
replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options,
|
||||
replication_object_is_ssec_encrypted, replication_put_object_header_size, replication_put_object_options,
|
||||
replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
|
||||
ssec_passthrough_gate, version_identity_drifted,
|
||||
ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -123,6 +125,7 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_
|
||||
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
|
||||
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
|
||||
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
|
||||
const EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED: &str = "replication_drifted_replica_located";
|
||||
const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
|
||||
const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
|
||||
|
||||
@@ -332,6 +335,12 @@ fn verify_single_part_replica(
|
||||
const REPLICA_ETAG_MISMATCH_ERROR: &str = "replica etag mismatch: the target persisted different bytes than were sent";
|
||||
|
||||
fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
|
||||
// Every write refreshes the cached verdict, so the convergence fallback
|
||||
// below (`replica_head_fallback`) knows whether a 404 on a
|
||||
// version-addressed HEAD can mean "replica missing" on this target.
|
||||
if let Some(capability) = version_identity_capability_from_put(source_version_id, assigned_version_id) {
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, capability);
|
||||
}
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
}
|
||||
@@ -404,11 +413,70 @@ async fn head_object_fallback(
|
||||
) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
|
||||
Ok(oi) => Ok(Some(oi)),
|
||||
Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
|
||||
Err(e) if head_object_not_found(&e) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
fn head_object_not_found(err: &HeadObjectSdkError) -> bool {
|
||||
err.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(err, 404)
|
||||
}
|
||||
|
||||
/// Second look at a replica whose version-addressed HEAD failed, for the two
|
||||
/// target shapes where that failure is not a verdict on the replica:
|
||||
///
|
||||
/// - AWS-style 400/403 (the RustFS uuid is rejected as malformed): HEAD the
|
||||
/// current version without a version id; callers compare ETags.
|
||||
/// - 404 on a target known to mint its own version ids (the Wasabi shape,
|
||||
/// rustfs/backlog#2340): the source id never existed there, so locate the
|
||||
/// replica by exact key and ETag through ListObjectVersions and HEAD the id
|
||||
/// the target assigned. Without this, every heal, MRF retry and
|
||||
/// existing-object resync re-drive PUTs the object again and mints one
|
||||
/// more target version.
|
||||
///
|
||||
/// `None` when the error stands as-is: a real miss on an adopting target, or
|
||||
/// a target whose identity contract is still unknown. A failed lookup is
|
||||
/// returned as a HEAD-shaped error so callers keep their "target operation
|
||||
/// failed" handling (retry later) instead of re-driving the PUT.
|
||||
async fn replica_head_fallback(
|
||||
tgt_client: &TargetClient,
|
||||
object: &str,
|
||||
source_etag: Option<&str>,
|
||||
err: &HeadObjectSdkError,
|
||||
) -> Option<std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError>> {
|
||||
if is_version_id_format_mismatch(err) {
|
||||
return Some(head_object_fallback(tgt_client, object).await);
|
||||
}
|
||||
if !head_object_not_found(err)
|
||||
|| !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let etag = source_etag.filter(|etag| !etag.trim().is_empty())?;
|
||||
Some(match tgt_client.find_version_by_etag(&tgt_client.bucket, object, etag).await {
|
||||
Ok(Some(assigned_version_id)) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %tgt_client.bucket,
|
||||
object = %object,
|
||||
arn = %tgt_client.arn,
|
||||
assigned_version_id = %assigned_version_id,
|
||||
"Located replica by content identity on a target that mints its own version ids"
|
||||
);
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, Some(assigned_version_id)).await {
|
||||
Ok(oi) => Ok(Some(oi)),
|
||||
// The located version disappeared between LIST and HEAD.
|
||||
Err(e) if head_object_not_found(&e) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
Ok(None) => Ok(None),
|
||||
Err(list_err) => Err(Box::new(SdkError::construction_failure(*list_err))),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve the N2 fail-closed gate for an SSE-C passthrough attempt against
|
||||
/// this target. Returns `Some(audit_required)` when replication may proceed;
|
||||
/// on a freshly-flagged header-dropping target it settles `rinfo` as FAILED
|
||||
@@ -1400,31 +1468,26 @@ async fn verify_resync_head_result(
|
||||
(0, None)
|
||||
}
|
||||
}
|
||||
Err(err) if is_version_id_format_mismatch(&err) => {
|
||||
// AWS-style target rejects the RustFS UUID versionId
|
||||
// (400). Re-verify without the versionId before
|
||||
// concluding the object failed to replicate, instead
|
||||
// of counting a well-replicated object as failed.
|
||||
match head_object_fallback(target_client.as_ref(), &roi.name).await {
|
||||
Ok(Some(_)) => {
|
||||
Err(err) => {
|
||||
// A version-addressed HEAD is not the last word on every target:
|
||||
// re-verify through the fallback before counting a well-replicated
|
||||
// object as failed (see `replica_head_fallback`).
|
||||
match replica_head_fallback(target_client.as_ref(), &roi.name, roi.etag.as_deref(), &err).await {
|
||||
Some(Ok(Some(_))) => {
|
||||
st.replicated_count += 1;
|
||||
st.replicated_size += roi.size;
|
||||
(roi.size, None)
|
||||
}
|
||||
Ok(None) => {
|
||||
Some(Ok(None)) | None => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(err))
|
||||
}
|
||||
Err(e2) => {
|
||||
Some(Err(e2)) => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(e2))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3597,11 +3660,8 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
|
||||
// Object not on target yet → fall through to PUT.
|
||||
} else if is_version_id_format_mismatch(&e) {
|
||||
// Version-ID format mismatch: retry without versionId and compare ETags.
|
||||
match head_object_fallback(&tgt_client, &object).await {
|
||||
if let Some(fallback) = replica_head_fallback(&tgt_client, &object, object_info.etag.as_deref(), &e).await {
|
||||
match fallback {
|
||||
Ok(Some(oi)) if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) => {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&oi, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
@@ -3631,6 +3691,8 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
} else if head_object_not_found(&e) {
|
||||
// Object not on target yet → fall through to PUT.
|
||||
} else {
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
@@ -4230,9 +4292,8 @@ async fn resolve_replicate_all_action(
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if is_version_id_format_mismatch(&e) {
|
||||
// Version-ID format mismatch: retry without versionId and compare ETags.
|
||||
match head_object_fallback(tgt_client, object).await {
|
||||
if let Some(fallback) = replica_head_fallback(tgt_client, object, object_info.etag.as_deref(), &e).await {
|
||||
match fallback {
|
||||
Ok(Some(oi)) => {
|
||||
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
|
||||
if require_existing_target && !etags_match {
|
||||
@@ -4284,7 +4345,7 @@ async fn resolve_replicate_all_action(
|
||||
return None;
|
||||
}
|
||||
}
|
||||
} else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
|
||||
} else if head_object_not_found(&e) {
|
||||
if require_existing_target {
|
||||
rinfo.error = Some("replica metadata target does not contain this object version".to_string());
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
@@ -4656,6 +4717,58 @@ where
|
||||
result
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MultipartReplicationReadPlan {
|
||||
part_number: i32,
|
||||
part_size: i64,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
next_offset: i64,
|
||||
}
|
||||
|
||||
fn multipart_replication_read_plan(
|
||||
object_info: &ObjectInfo,
|
||||
obj_opts: &ObjectOptions,
|
||||
mut input: ReplicationMultipartPartInput,
|
||||
stored_size: usize,
|
||||
is_last: bool,
|
||||
) -> std::io::Result<MultipartReplicationReadPlan> {
|
||||
let empty_last_part = is_last && input.part_size == 0 && stored_size == 0;
|
||||
// Raw reads address stored bytes. Only untransformed legacy parts may
|
||||
// substitute their stored size for a missing logical size.
|
||||
if obj_opts.raw_data_movement_read || (input.part_size == 0 && !object_info.is_compressed() && !object_info.is_encrypted()) {
|
||||
input.part_size = i64::try_from(stored_size).map_err(|_| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, "multipart replication stored part size exceeds i64")
|
||||
})?;
|
||||
}
|
||||
if empty_last_part {
|
||||
if input.offset < 0 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"empty multipart replication part has a negative offset",
|
||||
));
|
||||
}
|
||||
let part_number = i32::try_from(input.part_number)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "multipart replication part number exceeds i32"))?;
|
||||
return Ok(MultipartReplicationReadPlan {
|
||||
part_number,
|
||||
part_size: 0,
|
||||
range: None,
|
||||
next_offset: input.offset,
|
||||
});
|
||||
}
|
||||
let plan = replication_multipart_part_plan(input).map_err(std::io::Error::other)?;
|
||||
Ok(MultipartReplicationReadPlan {
|
||||
part_number: plan.part_number,
|
||||
part_size: plan.part_size,
|
||||
range: Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: plan.range.start,
|
||||
end: plan.range.end,
|
||||
}),
|
||||
next_offset: plan.next_offset,
|
||||
})
|
||||
}
|
||||
|
||||
async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
|
||||
ctx: MultipartReplicationContext<'_, S>,
|
||||
upload_id: &str,
|
||||
@@ -4676,35 +4789,31 @@ async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
|
||||
|
||||
let mut header_size = replication_put_object_header_size(&put_opts);
|
||||
let mut offset: i64 = 0;
|
||||
for part_info in object_info.parts.iter() {
|
||||
// Ciphertext passthrough (raw read) ranges over the stored part
|
||||
// bytes; decrypted reads range over the logical plaintext parts.
|
||||
let part_size = if obj_opts.raw_data_movement_read {
|
||||
part_info.size as i64
|
||||
} else {
|
||||
part_info.actual_size
|
||||
};
|
||||
let part_plan = replication_multipart_part_plan(ReplicationMultipartPartInput {
|
||||
offset,
|
||||
part_number: part_info.number,
|
||||
part_size,
|
||||
})
|
||||
.map_err(|err| std::io::Error::other(err.to_string()))?;
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: part_plan.range.start,
|
||||
end: part_plan.range.end,
|
||||
};
|
||||
for (index, part_info) in object_info.parts.iter().enumerate() {
|
||||
let part_plan = multipart_replication_read_plan(
|
||||
object_info,
|
||||
obj_opts,
|
||||
ReplicationMultipartPartInput {
|
||||
offset,
|
||||
part_number: part_info.number,
|
||||
part_size: part_info.actual_size,
|
||||
},
|
||||
part_info.size,
|
||||
index + 1 == object_info.parts.len(),
|
||||
)?;
|
||||
offset = part_plan.next_offset;
|
||||
|
||||
let part_reader = storage
|
||||
.get_object_reader(src_bucket, object, Some(range_spec), HeaderMap::new(), obj_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
|
||||
let part_stream = wrap_with_bandwidth_monitor_with_header(part_reader.stream, src_bucket, arn, header_size);
|
||||
let byte_stream = if let Some(range_spec) = part_plan.range {
|
||||
let part_reader = storage
|
||||
.get_object_reader(src_bucket, object, Some(range_spec), HeaderMap::new(), obj_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
let part_stream = wrap_with_bandwidth_monitor_with_header(part_reader.stream, src_bucket, arn, header_size);
|
||||
async_read_to_bytestream(part_stream)
|
||||
} else {
|
||||
ByteStream::from_static(b"")
|
||||
};
|
||||
header_size = 0;
|
||||
let byte_stream = async_read_to_bytestream(part_stream);
|
||||
|
||||
let object_part = cli
|
||||
.put_object_part(
|
||||
@@ -4760,6 +4869,173 @@ async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError;
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_preserves_legacy_plain_part_ranges() {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
size: 6 * 1024 * 1024,
|
||||
..Default::default()
|
||||
};
|
||||
let mut offset = 0;
|
||||
for (part_number, stored_size, start, end) in [
|
||||
(1, 5 * MIB, 0, 5 * 1024 * 1024 - 1),
|
||||
(2, MIB, 5 * 1024 * 1024, 6 * 1024 * 1024 - 1),
|
||||
] {
|
||||
let plan = multipart_replication_read_plan(
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
ReplicationMultipartPartInput {
|
||||
offset,
|
||||
part_number,
|
||||
part_size: 0,
|
||||
},
|
||||
stored_size,
|
||||
part_number == 2,
|
||||
)
|
||||
.expect("legacy plain parts must use their stored sizes");
|
||||
assert_eq!(plan.part_number, i32::try_from(part_number).expect("part number fits"));
|
||||
assert_eq!(plan.part_size, i64::try_from(stored_size).expect("stored size fits"));
|
||||
let range = plan.range.expect("a nonempty part must read a range");
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!((range.start, range.end), (start, end));
|
||||
assert_eq!(plan.next_offset, end + 1);
|
||||
offset = plan.next_offset;
|
||||
}
|
||||
assert_eq!(offset, object_info.size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_distinguishes_transformed_and_raw_sizes() {
|
||||
for metadata in [
|
||||
HashMap::from([("x-rustfs-internal-compression".to_string(), "klauspost/compress/s2".to_string())]),
|
||||
HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())]),
|
||||
] {
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(object_info.is_compressed() || object_info.is_encrypted());
|
||||
for raw in [false, true] {
|
||||
for actual_size in [-1, 0, 5] {
|
||||
let result = multipart_replication_read_plan(
|
||||
&object_info,
|
||||
&ObjectOptions {
|
||||
raw_data_movement_read: raw,
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationMultipartPartInput {
|
||||
offset: 7,
|
||||
part_number: 2,
|
||||
part_size: actual_size,
|
||||
},
|
||||
9,
|
||||
true,
|
||||
);
|
||||
if !raw && actual_size <= 0 {
|
||||
let err = result.expect_err("transformed reads cannot substitute physical bytes for unknown plaintext");
|
||||
assert!(matches!(
|
||||
err.get_ref().and_then(|err| err.downcast_ref::<ReplicationMultipartPlanError>()),
|
||||
Some(ReplicationMultipartPlanError::InvalidPartSize { part_size })
|
||||
if *part_size == actual_size
|
||||
));
|
||||
} else {
|
||||
let plan = result.expect("the selected representation has a known positive size");
|
||||
let expected_size = if raw { 9 } else { 5 };
|
||||
assert_eq!(plan.part_number, 2);
|
||||
assert_eq!(plan.part_size, expected_size);
|
||||
let range = plan.range.expect("a nonempty part must read a range");
|
||||
assert_eq!((range.start, range.end), (7, 7 + expected_size - 1));
|
||||
assert_eq!(plan.next_offset, 7 + expected_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_retains_an_empty_last_part_without_advancing() {
|
||||
for offset in [5 * 1024 * 1024, i64::MAX] {
|
||||
for raw in [false, true] {
|
||||
let plan = multipart_replication_read_plan(
|
||||
&ObjectInfo::default(),
|
||||
&ObjectOptions {
|
||||
raw_data_movement_read: raw,
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationMultipartPartInput {
|
||||
offset,
|
||||
part_number: 2,
|
||||
part_size: 0,
|
||||
},
|
||||
0,
|
||||
true,
|
||||
)
|
||||
.expect("an empty final part needs no range read");
|
||||
assert_eq!(plan.part_number, 2);
|
||||
assert_eq!(plan.part_size, 0);
|
||||
assert!(plan.range.is_none());
|
||||
assert_eq!(plan.next_offset, offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_rejects_invalid_empty_parts_and_ranges() {
|
||||
for (offset, part_number, actual_size, stored_size, is_last) in [
|
||||
(0, 1, 0, 0, false),
|
||||
(0, 2, -1, 0, true),
|
||||
(0, 2, -1, 9, true),
|
||||
(-1, 2, 0, 0, true),
|
||||
(0, usize::try_from(i32::MAX).expect("i32 fits usize") + 1, 0, 0, true),
|
||||
(i64::MAX, 2, 1, 1, true),
|
||||
(i64::MAX, 2, 2, 2, true),
|
||||
] {
|
||||
let err = multipart_replication_read_plan(
|
||||
&ObjectInfo::default(),
|
||||
&ObjectOptions::default(),
|
||||
ReplicationMultipartPartInput {
|
||||
offset,
|
||||
part_number,
|
||||
part_size: actual_size,
|
||||
},
|
||||
stored_size,
|
||||
is_last,
|
||||
)
|
||||
.expect_err("invalid part metadata must not become a successful transport plan");
|
||||
assert!(
|
||||
err.kind() == std::io::ErrorKind::InvalidData
|
||||
|| err.get_ref().is_some_and(|err| { err.is::<ReplicationMultipartPlanError>() }),
|
||||
"the failure must preserve a typed metadata or planner error: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
#[test]
|
||||
fn multipart_read_plan_rejects_physical_size_overflow() {
|
||||
for raw in [false, true] {
|
||||
let err = multipart_replication_read_plan(
|
||||
&ObjectInfo::default(),
|
||||
&ObjectOptions {
|
||||
raw_data_movement_read: raw,
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationMultipartPartInput {
|
||||
offset: 0,
|
||||
part_number: 1,
|
||||
part_size: 0,
|
||||
},
|
||||
usize::MAX,
|
||||
true,
|
||||
)
|
||||
.expect_err("a physical size outside the range API must be rejected before casting");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "multipart replication stored part size exceeds i64");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_state_terminal_retry_uses_validate_only() {
|
||||
@@ -5168,6 +5444,178 @@ mod tests {
|
||||
ReplicationTargetStore::register_test_target(target).await;
|
||||
}
|
||||
|
||||
const DRIFTED_ASSIGNED_VERSION_ID: &str = "001788697733811332140-fR6j6uXKV-";
|
||||
const DRIFTED_ETAG: &str = "9a0364b9e99bb480dd25e1f0284c8555";
|
||||
|
||||
/// The Wasabi shape (rustfs/backlog#2340): a version-addressed HEAD with
|
||||
/// the source uuid answers 404 (not the AWS 400), ListObjectVersions shows
|
||||
/// the id the target minted, and a HEAD by that id succeeds. Serves exactly
|
||||
/// `requests` connections and returns the request lines it saw.
|
||||
fn spawn_drifted_target_server(requests: usize) -> (String, std::thread::JoinHandle<Vec<String>>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
|
||||
let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
|
||||
let handle = std::thread::spawn(move || {
|
||||
let mut seen = Vec::new();
|
||||
for _ in 0..requests {
|
||||
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
|
||||
let mut request = [0_u8; 8192];
|
||||
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
|
||||
let text = String::from_utf8_lossy(&request[..bytes_read]).to_string();
|
||||
let request_line = text.lines().next().unwrap_or_default().to_string();
|
||||
let response = if request_line.starts_with("HEAD ") {
|
||||
if request_line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}")) {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nETag: \"{DRIFTED_ETAG}\"\r\nContent-Length: 4\r\nLast-Modified: Sun, 06 Sep 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
} else {
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()
|
||||
}
|
||||
} else if request_line.starts_with("GET ") && request_line.contains("versions") {
|
||||
let body = format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>target-bucket</Name><Prefix>object</Prefix><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Version><Key>object</Key><VersionId>{DRIFTED_ASSIGNED_VERSION_ID}</VersionId><IsLatest>true</IsLatest><LastModified>2026-09-06T10:00:00.000Z</LastModified><ETag>"{DRIFTED_ETAG}"</ETag><Size>4</Size><StorageClass>STANDARD</StorageClass></Version></ListVersionsResult>"
|
||||
);
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
} else {
|
||||
"HTTP/1.1 500 Unexpected\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()
|
||||
};
|
||||
stream
|
||||
.write_all(response.as_bytes())
|
||||
.expect("test HTTP response should be written");
|
||||
seen.push(request_line);
|
||||
}
|
||||
seen
|
||||
});
|
||||
(endpoint, handle)
|
||||
}
|
||||
|
||||
fn drifted_roi_and_object() -> (ReplicateObjectInfo, ObjectInfo) {
|
||||
let roi = ReplicateObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
op_type: ReplicationType::Heal,
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let object_info = ObjectInfo {
|
||||
bucket: roi.bucket.clone(),
|
||||
name: roi.name.clone(),
|
||||
version_id: roi.version_id,
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
size: 4,
|
||||
..Default::default()
|
||||
};
|
||||
(roi, object_info)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_locates_replica_by_etag_on_target_that_mints_own_version_ids() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
roi: &roi,
|
||||
tgt_client: &target,
|
||||
bucket: &roi.bucket,
|
||||
object: &roi.name,
|
||||
start_time: OffsetDateTime::now_utc(),
|
||||
ssec_audit_required: false,
|
||||
},
|
||||
object_info,
|
||||
&mut rinfo,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(action, Some((ReplicationAction::None, _))),
|
||||
"a replica located by content identity must not be re-driven: {action:?}"
|
||||
);
|
||||
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 3, "HEAD by source id, ListObjectVersions, HEAD by assigned id: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD ") && seen[0].contains(&roi.version_id.unwrap().to_string()));
|
||||
assert!(seen[1].starts_with("GET ") && seen[1].contains("prefix=object"), "{}", seen[1]);
|
||||
assert!(seen[2].starts_with("HEAD ") && seen[2].contains(DRIFTED_ASSIGNED_VERSION_ID));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_not_found_still_replicates_when_identity_contract_is_unknown() {
|
||||
// Same 404, but the target never revealed whether it adopts version
|
||||
// ids: a 404 keeps meaning "replica missing" (adopting targets, e.g.
|
||||
// RustFS/MinIO peers, must not skip a genuinely missing version).
|
||||
let (endpoint, server) = spawn_head_status_server(404);
|
||||
let target = test_target_client(endpoint);
|
||||
let (roi, object_info) = drifted_roi_and_object();
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
roi: &roi,
|
||||
tgt_client: &target,
|
||||
bucket: &roi.bucket,
|
||||
object: &roi.name,
|
||||
start_time: OffsetDateTime::now_utc(),
|
||||
ssec_audit_required: false,
|
||||
},
|
||||
object_info,
|
||||
&mut rinfo,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(action, Some((ReplicationAction::All, _))));
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resync_verification_counts_drifted_replica_as_replicated() {
|
||||
let (endpoint, server) = spawn_drifted_target_server(3);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, _) = drifted_roi_and_object();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (4, 1, 0));
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_response_audit_records_identity_verdict() {
|
||||
let target = test_target_client("http://127.0.0.1:1".to_string());
|
||||
let source = Uuid::new_v4().to_string();
|
||||
audit_target_version_identity(&target, &source, Some(DRIFTED_ASSIGNED_VERSION_ID));
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
audit_target_version_identity(&target, &source, Some(&source));
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
// An unversioned write carries no contract and must not overwrite it.
|
||||
audit_target_version_identity(&target, "null", None);
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::Adopts
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_admission_configuration_is_bounded() {
|
||||
assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");
|
||||
@@ -6339,4 +6787,326 @@ mod tests {
|
||||
"one target's report must not silence another's"
|
||||
);
|
||||
}
|
||||
mod multipart_transport_tests {
|
||||
use super::super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||
use super::super::super::replication_storage_boundary::ObjectIO as _;
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use std::convert::Infallible;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Source {
|
||||
body: Bytes,
|
||||
info: ObjectInfo,
|
||||
ranges: StdMutex<Vec<(i64, i64)>>,
|
||||
full_reads: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl super::super::super::replication_storage_boundary::ObjectIO for Source {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = super::super::super::replication_storage_boundary::PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
_headers: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
assert_eq!(
|
||||
opts.version_id,
|
||||
self.info.version_id.map(|id| id.to_string()),
|
||||
"every read retains the selected source version"
|
||||
);
|
||||
if range.is_none() {
|
||||
self.full_reads.fetch_add(1, Ordering::Relaxed);
|
||||
return Ok(GetObjectReader {
|
||||
stream: Box::new(std::io::Cursor::new(self.body.clone())),
|
||||
object_info: self.info.clone(),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
let range = range.expect("multipart transport must request an explicit nonempty range");
|
||||
assert!(!range.is_suffix_length);
|
||||
assert!(range.start <= range.end, "empty parts must not issue an inverted range");
|
||||
self.ranges.lock().expect("range journal lock").push((range.start, range.end));
|
||||
let start = usize::try_from(range.start).expect("nonnegative start");
|
||||
let end = usize::try_from(range.end).expect("nonnegative end");
|
||||
let body = self.body.slice(start..=end);
|
||||
Ok(GetObjectReader {
|
||||
stream: Box::new(std::io::Cursor::new(body)),
|
||||
object_info: self.info.clone(),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
panic!("replication must not overwrite its source")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RequestRecord {
|
||||
method: http::Method,
|
||||
query: HashMap<String, String>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_legacy_zero_actual_sizes() {
|
||||
run_transport(4096, None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_uploads_an_empty_last_part_without_reading_a_range() {
|
||||
run_transport(0, None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_transformed_unknown_nonempty_parts() {
|
||||
for unknown_part in [(0, 0), (1, 0), (0, -1), (1, -1)] {
|
||||
run_transport(4096, Some(unknown_part)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_transformed_empty_tail() {
|
||||
run_transport(0, Some((1, 0))).await;
|
||||
}
|
||||
|
||||
async fn run_transport(tail_size: usize, unknown_part: Option<(usize, i64)>) {
|
||||
const FIRST_SIZE: usize = 5 * 1024 * 1024;
|
||||
let body = Bytes::from([vec![0x35; FIRST_SIZE], vec![0xa7; tail_size]].concat());
|
||||
let etag = faster_hex::hex_string(rustfs_utils::hash::HashAlgorithm::Md5.hash_encode(&body).as_ref());
|
||||
let source = Arc::new(Source {
|
||||
info: ObjectInfo {
|
||||
size: i64::try_from(body.len() + if unknown_part.is_some() { 16 } else { 0 }).expect("stored size"),
|
||||
actual_size: i64::try_from(body.len()).expect("body size"),
|
||||
etag: Some(etag.clone()),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
user_defined: Arc::new(if unknown_part.is_some() {
|
||||
HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])
|
||||
} else {
|
||||
HashMap::new()
|
||||
}),
|
||||
parts: Arc::new(vec![
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: FIRST_SIZE + if unknown_part.is_some() { 8 } else { 0 },
|
||||
actual_size: if let Some((0, size)) = unknown_part {
|
||||
size
|
||||
} else if unknown_part.is_some() || tail_size == 0 {
|
||||
i64::try_from(FIRST_SIZE).expect("first part size")
|
||||
} else {
|
||||
0
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
number: 2,
|
||||
size: tail_size + if unknown_part.is_some() { 8 } else { 0 },
|
||||
actual_size: if let Some((1, size)) = unknown_part {
|
||||
size
|
||||
} else if unknown_part.is_some() {
|
||||
i64::try_from(tail_size).expect("tail logical size")
|
||||
} else {
|
||||
0
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
body: body.clone(),
|
||||
ranges: StdMutex::new(Vec::new()),
|
||||
full_reads: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
let journal = Arc::new(StdMutex::new(Vec::<RequestRecord>::new()));
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
|
||||
.await
|
||||
.expect("bind multipart target");
|
||||
let endpoint = format!("http://{}", listener.local_addr().expect("multipart target address"));
|
||||
let server_journal = journal.clone();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
let (stream, _) = listener.accept().await.expect("accept multipart request");
|
||||
let journal = server_journal.clone();
|
||||
connections.spawn(async move {
|
||||
let service = hyper::service::service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
|
||||
let journal = journal.clone();
|
||||
async move {
|
||||
let (request, body) = request.into_parts();
|
||||
let query: HashMap<String, String> = url::form_urlencoded::parse(
|
||||
request.uri.query().unwrap_or_default().as_bytes(),
|
||||
).into_owned().collect();
|
||||
let body = body.collect().await.expect("read complete multipart request body").to_bytes();
|
||||
let response = if request.method == http::Method::POST && query.contains_key("uploads") {
|
||||
"<InitiateMultipartUploadResult><Bucket>target-bucket</Bucket><Key>object</Key><UploadId>upload-1</UploadId></InitiateMultipartUploadResult>"
|
||||
} else if request.method == http::Method::PUT {
|
||||
""
|
||||
} else if request.method == http::Method::POST && query.contains_key("uploadId") {
|
||||
"<CompleteMultipartUploadResult><Location>http://localhost/object</Location><Bucket>target-bucket</Bucket><Key>object</Key><ETag>"target-2"</ETag></CompleteMultipartUploadResult>"
|
||||
} else if request.method == http::Method::DELETE && query.contains_key("uploadId") {
|
||||
""
|
||||
} else {
|
||||
panic!("unexpected multipart request: {} {}", request.method, request.uri)
|
||||
};
|
||||
let response_etag = if request.method == http::Method::PUT && !query.contains_key("partNumber") {
|
||||
format!("\"{}\"", faster_hex::hex_string(rustfs_utils::hash::HashAlgorithm::Md5.hash_encode(&body).as_ref()))
|
||||
} else {
|
||||
"\"uploaded-part\"".to_string()
|
||||
};
|
||||
journal.lock().expect("request journal lock").push(RequestRecord {
|
||||
method: request.method, query, headers: request.headers, body,
|
||||
});
|
||||
Ok::<_, Infallible>(hyper::Response::builder()
|
||||
.header("content-type", "application/xml")
|
||||
.header("etag", response_etag)
|
||||
.body(Full::new(Bytes::from_static(response.as_bytes())))
|
||||
.expect("multipart response"))
|
||||
}
|
||||
});
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(hyper_util::rt::TokioIo::new(stream), service)
|
||||
.await.expect("serve multipart connection");
|
||||
});
|
||||
}
|
||||
});
|
||||
let mut target = test_target_client(endpoint);
|
||||
let config = target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired)
|
||||
.force_path_style(true)
|
||||
.build();
|
||||
Arc::get_mut(&mut target).expect("unshared test target").client = Arc::new(aws_sdk_s3::Client::from_conf(config));
|
||||
let (put_opts, is_multipart) = replication_put_object_options("STANDARD", &source.info).expect("replication options");
|
||||
let opts = ObjectOptions {
|
||||
version_id: source.info.version_id.map(|id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let reader = source
|
||||
.get_object_reader("source", "object", None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("open the existing full-object stream");
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
replicate_all_payload_to_target(
|
||||
ReplicateAllPayloadContext {
|
||||
storage: &source,
|
||||
tgt_client: &target,
|
||||
bucket: "source",
|
||||
object: "object",
|
||||
object_info: &source.info,
|
||||
obj_opts: &opts,
|
||||
arn: &target.arn,
|
||||
transfer_size: i64::try_from(body.len()).expect("plaintext size"),
|
||||
is_multipart,
|
||||
put_opts,
|
||||
},
|
||||
reader,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
server.abort();
|
||||
assert!(server.await.expect_err("fixture server is stopped").is_cancelled());
|
||||
if let Some(error) = result.expect("replication must finish") {
|
||||
panic!("legacy parts must replicate successfully: {error}");
|
||||
}
|
||||
assert_eq!(
|
||||
source.full_reads.load(Ordering::Relaxed),
|
||||
1,
|
||||
"reuse the initial full stream without an extra read"
|
||||
);
|
||||
if unknown_part.is_some() {
|
||||
let requests = journal.lock().expect("request journal lock");
|
||||
assert_eq!(requests.len(), 1, "unknown transformed boundaries retain one streaming PUT");
|
||||
let request = &requests[0];
|
||||
assert_eq!(request.method, http::Method::PUT);
|
||||
let source_version = source.info.version_id.map(|id| id.to_string()).expect("versioned fixture");
|
||||
assert_eq!(
|
||||
request.query,
|
||||
HashMap::from([
|
||||
("x-id".to_string(), "PutObject".to_string()),
|
||||
("versionId".to_string(), source_version.clone()),
|
||||
]),
|
||||
"single PUT carries only the SDK operation query and the source versionId the target must reuse"
|
||||
);
|
||||
assert_eq!(request.body, body, "single PUT includes every byte of both source parts");
|
||||
assert_eq!(
|
||||
request.headers.get("content-length").expect("body length"),
|
||||
body.len().to_string().as_str()
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&request.headers, rustfs_utils::http::SUFFIX_SOURCE_ETAG).as_deref(),
|
||||
Some(etag.as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&request.headers, rustfs_utils::http::SUFFIX_SOURCE_VERSION_ID)
|
||||
.map(|value| value.into_owned()),
|
||||
Some(source_version),
|
||||
"single PUT preserves the selected source version"
|
||||
);
|
||||
assert!(
|
||||
source.ranges.lock().expect("range journal lock").is_empty(),
|
||||
"unknown logical boundaries must not issue guessed ranges"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let requests = journal.lock().expect("request journal lock");
|
||||
assert_eq!(requests.len(), 4, "initiate, two upload parts, and complete without retries");
|
||||
assert!(requests[0].query.contains_key("uploads"));
|
||||
for (index, expected) in [(1, body.slice(..FIRST_SIZE)), (2, body.slice(FIRST_SIZE..))] {
|
||||
assert_eq!(requests[index].method, http::Method::PUT);
|
||||
assert_eq!(requests[index].query.get("partNumber"), Some(&index.to_string()));
|
||||
assert_eq!(requests[index].body, expected, "upload part contains the exact source range");
|
||||
assert_eq!(
|
||||
requests[index].headers.get("content-length").expect("part content length"),
|
||||
expected.len().to_string().as_str()
|
||||
);
|
||||
}
|
||||
let complete = &requests[3];
|
||||
assert_eq!(complete.method, http::Method::POST);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&complete.headers, rustfs_utils::http::SUFFIX_SOURCE_ETAG).as_deref(),
|
||||
Some(etag.as_str())
|
||||
);
|
||||
let complete_xml = std::str::from_utf8(&complete.body).expect("complete XML");
|
||||
assert_eq!(
|
||||
complete_xml.matches("<Part>").count(),
|
||||
2,
|
||||
"the empty final part must remain in the completion list"
|
||||
);
|
||||
assert!(complete_xml.contains("<PartNumber>1</PartNumber>"));
|
||||
assert!(complete_xml.contains("<PartNumber>2</PartNumber>"));
|
||||
let mut expected_ranges = vec![(0, i64::try_from(FIRST_SIZE - 1).expect("first end"))];
|
||||
if tail_size > 0 {
|
||||
expected_ranges.push((
|
||||
i64::try_from(FIRST_SIZE).expect("tail start"),
|
||||
i64::try_from(body.len() - 1).expect("tail end"),
|
||||
));
|
||||
}
|
||||
assert_eq!(*source.ranges.lock().expect("range journal lock"), expected_ranges);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
};
|
||||
pub use rustfs_replication::{VersionIdentityCapability, version_identity_capability_from_put};
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -192,6 +193,14 @@ impl ReplicationTargetStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
|
||||
BucketTargetSys::get().version_identity_capability(arn)
|
||||
}
|
||||
|
||||
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
|
||||
BucketTargetSys::get().record_version_identity_capability(arn, capability)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -248,7 +257,16 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string());
|
||||
}
|
||||
|
||||
let mut is_multipart = object_info.is_multipart();
|
||||
// Older transformed objects can have physical parts without logical part
|
||||
// lengths. Keep their existing whole-object transport: physical sizes are
|
||||
// not plaintext boundaries for a multipart replication read.
|
||||
let legacy_single_put = object_info.etag.as_deref().is_none_or(|etag| etag.len() == 32);
|
||||
let base_is_multipart = object_info.is_multipart()
|
||||
&& !(legacy_single_put
|
||||
&& object_info.parts.len() > 1
|
||||
&& (object_info.is_compressed() || object_info.is_encrypted())
|
||||
&& object_info.parts.iter().any(|part| part.actual_size <= 0));
|
||||
let mut is_multipart = base_is_multipart;
|
||||
|
||||
if let Some(checksum_data) = &object_info.checksum
|
||||
&& !checksum_data.is_empty()
|
||||
@@ -259,8 +277,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
} else if object_info.is_encrypted() {
|
||||
// Encrypted checksums cannot be exposed as plaintext headers, and
|
||||
// decrypt_checksums reports is_multipart=false for them (a value
|
||||
// the response path relies on). Keep the object's own multipart
|
||||
// flag so encrypted objects stay on the multipart route.
|
||||
// the response path relies on). Keep the transport selected from
|
||||
// the object's layout and readable part boundaries.
|
||||
} else {
|
||||
let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
|
||||
// The checksum record describes how the *checksum* is composed,
|
||||
@@ -268,23 +286,37 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
// MULTIPART flag even on a multipart upload, so trusting it here
|
||||
// routed a 768-part object through a single PutObject and the
|
||||
// target rejected the 6 GiB body with EntityTooLarge
|
||||
// (rustfs#6825). The object's own shape is the authority: the
|
||||
// (rustfs#6825). The usable part layout is the authority: the
|
||||
// record may only add multipart-ness, never take it away.
|
||||
is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
|
||||
is_multipart = base_is_multipart || checksum_record_is_multipart;
|
||||
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key != AMZ_CHECKSUM_TYPE {
|
||||
meta.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !object_info.is_multipart()
|
||||
if !base_is_multipart
|
||||
&& checksum_meta
|
||||
.get(AMZ_CHECKSUM_TYPE)
|
||||
.is_some_and(|value| value == AMZ_CHECKSUM_TYPE_FULL_OBJECT)
|
||||
{
|
||||
is_multipart = false;
|
||||
}
|
||||
|
||||
// The record keys each checksum by algorithm name ("CRC32"); the
|
||||
// target only reads `x-amz-checksum-<algorithm>`. Inserting the bare
|
||||
// name here made `PutObjectOptions::header()` send it as user
|
||||
// metadata (`x-amz-meta-crc32`), so no replica ever carried the
|
||||
// source checksum (rustfs/backlog#2340). The object-level record
|
||||
// describes one PUT body: a multipart replica is rebuilt part by
|
||||
// part, and its CreateMultipartUpload must not announce a checksum
|
||||
// the parts do not carry, so the record is forwarded on the
|
||||
// single-PUT route only (MinIO `getCRCMeta` parity).
|
||||
if !is_multipart {
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key == AMZ_CHECKSUM_TYPE {
|
||||
continue;
|
||||
}
|
||||
if let Some(header) = rustfs_rio::ChecksumType::from_string(key).key() {
|
||||
meta.insert(header.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -516,6 +548,7 @@ fn is_standard_header(key: &str) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||
use super::*;
|
||||
use aws_smithy_types::DateTime;
|
||||
use rustfs_replication::content_matches_by_etag;
|
||||
@@ -550,6 +583,109 @@ mod tests {
|
||||
checksum.to_bytes(&combined)
|
||||
}
|
||||
|
||||
fn replication_route_metadata() -> [(&'static str, Arc<HashMap<String, String>>); 4] {
|
||||
let mut compressed = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut compressed, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
[
|
||||
("plain", Arc::new(HashMap::new())),
|
||||
("compressed", Arc::new(compressed)),
|
||||
(
|
||||
"encrypted",
|
||||
Arc::new(HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
(
|
||||
"ssec",
|
||||
Arc::new(HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn replication_route_object(
|
||||
etag: Option<&str>,
|
||||
actual_sizes: [i64; 3],
|
||||
metadata: Arc<HashMap<String, String>>,
|
||||
) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
size: 48,
|
||||
actual_size: 12,
|
||||
user_defined: metadata,
|
||||
parts: Arc::new(
|
||||
actual_sizes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, actual_size)| ObjectPartInfo {
|
||||
number: index + 1,
|
||||
size: 16,
|
||||
actual_size,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transformed_single_put_parts_keep_the_previous_replication_route() {
|
||||
let [_, (_, compressed), (_, encrypted), (_, ssec)] = replication_route_metadata();
|
||||
let cases = [
|
||||
(
|
||||
"compressed middle zero",
|
||||
compressed.clone(),
|
||||
Some("0123456789abcdef0123456789abcdef"),
|
||||
[4, 0, 4],
|
||||
),
|
||||
("compressed tail unknown", compressed, None, [4, 4, -1]),
|
||||
(
|
||||
"encrypted middle unknown",
|
||||
encrypted,
|
||||
Some("gggggggggggggggggggggggggggggggg"),
|
||||
[4, -1, 4],
|
||||
),
|
||||
("ssec tail zero", ssec.clone(), None, [4, 4, 0]),
|
||||
("ssec middle unknown", ssec, Some("gggggggggggggggggggggggggggggggg"), [4, -1, 4]),
|
||||
];
|
||||
for (name, metadata, etag, actual_sizes) in cases {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let mut object_info = replication_route_object(etag, actual_sizes, metadata.clone());
|
||||
object_info.checksum = checksum;
|
||||
assert!(object_info.is_multipart(), "{name}: physical parts remain visible to metadata APIs");
|
||||
assert!(object_info.is_compressed() || object_info.is_encrypted());
|
||||
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("legacy transformed put options");
|
||||
assert!(
|
||||
!is_multipart,
|
||||
"{name}: unknown logical part sizes must preserve the old whole-object route"
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
if metadata.contains_key(SSEC_ALGORITHM_HEADER) {
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_SSEC_CRC).is_some(),
|
||||
object_info.checksum.is_some(),
|
||||
"SSE-C checksums retain their raw passthrough transport"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positive_part_sizes_and_legacy_multipart_etags_keep_the_replication_route() {
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
for (etag, actual_sizes) in [
|
||||
("0123456789abcdef0123456789abcdef", [4, 4, 4]),
|
||||
("0123456789abcdef0123456789abcdef-3", [4, 0, -1]),
|
||||
] {
|
||||
let mut object_info = replication_route_object(Some(etag), actual_sizes, metadata.clone());
|
||||
object_info.checksum = Some(full_object_multipart_checksum_record());
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("multipart put options");
|
||||
assert!(is_multipart, "{name}/{etag}: usable sizes and old multipart ETags must retain MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
|
||||
// rustfs#6825: a 768-part upload was replicated with a single
|
||||
@@ -582,6 +718,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_multipart_parts_keep_the_replication_route_without_a_multipart_etag() {
|
||||
for etag in [Some("0123456789abcdef0123456789abcdef"), None] {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
checksum,
|
||||
parts: Arc::new(
|
||||
(1..=2)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(
|
||||
is_multipart,
|
||||
"stored parts must retain multipart routing: etag={etag:?}, checksum={:?}",
|
||||
object_info.checksum
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
|
||||
// The mirror of the rustfs#6825 guard: an object stored as one PUT
|
||||
@@ -592,6 +758,10 @@ mod tests {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -628,6 +798,19 @@ mod tests {
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
|
||||
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
let mut legacy = replication_route_object(Some("0123456789abcdef0123456789abcdef"), [4, 0, 4], metadata);
|
||||
legacy.checksum = Some(checksum.to_bytes(&combined));
|
||||
let (_, record_is_multipart) = legacy.decrypt_checksums(0, &HeaderMap::new()).expect("decode checksum");
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &legacy).expect("legacy checksum put options");
|
||||
if legacy.is_encrypted() {
|
||||
assert!(!is_multipart, "{name}: encrypted checksum records must not change the old transport");
|
||||
} else {
|
||||
assert!(record_is_multipart, "the composite checksum must carry its own multipart signal");
|
||||
assert!(is_multipart, "{name}: a composite record can still promote the legacy route to MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1308,12 +1491,63 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (opts, _is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
|
||||
assert!(!is_multipart, "{name}: a single-part checksum record must keep the single-PUT route");
|
||||
let header = ty.key().expect("every forwarded algorithm has an x-amz-checksum header");
|
||||
assert_eq!(
|
||||
opts.user_metadata.get(name),
|
||||
opts.user_metadata.get(header),
|
||||
Some(&checksum.encoded),
|
||||
"replication must forward the {name} checksum into user_metadata identically to the classic algorithms"
|
||||
"replication must forward the {name} checksum as the {header} header"
|
||||
);
|
||||
assert!(
|
||||
!opts.user_metadata.contains_key(name),
|
||||
"{name}: the bare algorithm name would leave as x-amz-meta user metadata"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The object-level record of a multipart upload (composite or full-object)
|
||||
/// must not become a PutObject checksum header: the replica is rebuilt
|
||||
/// through CreateMultipartUpload/UploadPart, and a checksum announced there
|
||||
/// that the parts do not carry would be rejected by the target.
|
||||
#[test]
|
||||
fn replication_put_object_options_keeps_multipart_checksum_records_off_the_wire() {
|
||||
let mut composite_type = rustfs_rio::ChecksumType::from_string("crc32");
|
||||
composite_type
|
||||
.merge(rustfs_rio::ChecksumType::MULTIPART)
|
||||
.merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
|
||||
let mut combined = Vec::new();
|
||||
for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
|
||||
let part_checksum =
|
||||
rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
|
||||
combined.extend_from_slice(part_checksum.raw.as_slice());
|
||||
}
|
||||
let composite = rustfs_rio::Checksum::new_from_data(composite_type, &combined)
|
||||
.expect("composite checksum")
|
||||
.to_bytes(&combined);
|
||||
|
||||
for (label, checksum, etag) in [
|
||||
("composite", composite, "0123456789abcdef0123456789abcdef-2"),
|
||||
(
|
||||
"full-object",
|
||||
full_object_multipart_checksum_record(),
|
||||
"0123456789abcdef0123456789abcdef-3",
|
||||
),
|
||||
] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some(etag.to_string()),
|
||||
checksum: Some(checksum),
|
||||
..Default::default()
|
||||
};
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
assert!(is_multipart, "{label}: a multipart object must keep the multipart route");
|
||||
assert!(
|
||||
opts.user_metadata
|
||||
.keys()
|
||||
.all(|key| !key.starts_with("x-amz-checksum-") && key != "CRC32"),
|
||||
"{label}: no object-level checksum may reach the target's CreateMultipartUpload: {:?}",
|
||||
opts.user_metadata
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,8 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -49,10 +49,11 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
|
||||
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
|
||||
SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
|
||||
TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -92,6 +93,7 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -192,12 +194,102 @@ pub struct ScannerPeerActivity {
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageSnapshot {
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub generation: u64,
|
||||
pub pending_bucket_count: u64,
|
||||
pub protocol_version: u32,
|
||||
pub complete: bool,
|
||||
pub buckets: BTreeMap<String, u64>,
|
||||
pub buckets: BTreeMap<String, ScannerPeerDirtyUsageBucket>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageBucket {
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerScopedDirtyUsageAckEntry {
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerDirtyUsageAcknowledgement {
|
||||
Generation {
|
||||
host: String,
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
},
|
||||
Scoped {
|
||||
host: String,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payloads(
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<Vec<ScannerScopedDirtyUsageAckRequest>> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
if entries.is_empty() {
|
||||
return Err(Error::other("scoped dirty usage acknowledgement entries must be nonempty"));
|
||||
}
|
||||
|
||||
let mut payloads = Vec::with_capacity(entries.len().div_ceil(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize));
|
||||
let mut batch = Vec::with_capacity(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
for entry in entries {
|
||||
batch.push(ScannerScopedDirtyUsageEntry {
|
||||
bucket: entry.bucket,
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
generation: entry.generation,
|
||||
});
|
||||
if batch.len() == SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(
|
||||
&owner_id,
|
||||
&instance_id,
|
||||
probe_only,
|
||||
std::mem::take(&mut batch),
|
||||
)?);
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(&owner_id, &instance_id, probe_only, batch)?);
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payload(
|
||||
owner_id: &str,
|
||||
instance_id: &str,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<ScannerScopedDirtyUsageAckRequest> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
instance_id: instance_id.to_string(),
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only,
|
||||
entries,
|
||||
};
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -351,6 +443,11 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if !scanner_instance_id_is_valid(&response.instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot owner"))?;
|
||||
if response.generation == u64::MAX {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
|
||||
}
|
||||
@@ -386,9 +483,14 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if bucket.generation == 0 || bucket.generation > response.generation {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
|
||||
}
|
||||
Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.ok()
|
||||
.filter(|bucket_incarnation| !bucket_incarnation.is_nil())
|
||||
.ok_or_else(|| Error::other("peer scanner dirty usage snapshot contains an invalid bucket incarnation"))?;
|
||||
}
|
||||
|
||||
Ok(ScannerPeerDirtyUsageSnapshot {
|
||||
owner_id,
|
||||
instance_id: response.instance_id,
|
||||
generation: response.generation,
|
||||
pending_bucket_count: response.pending_bucket_count,
|
||||
@@ -397,7 +499,16 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
buckets: response
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket, bucket.generation))
|
||||
.map(|bucket| {
|
||||
(
|
||||
bucket.bucket,
|
||||
ScannerPeerDirtyUsageBucket {
|
||||
bucket_incarnation: Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.expect("bucket incarnation was validated"),
|
||||
generation: bucket.generation,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
@@ -1688,6 +1799,24 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_transition_transaction_compaction(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::transition_transaction_compaction_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
|
||||
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
|
||||
@@ -2068,19 +2197,10 @@ impl PeerRestClient {
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id, true, entries)?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2088,26 +2208,106 @@ impl PeerRestClient {
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical =
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
if !response.supported {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
.map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
|| !response.supported
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
Ok(())
|
||||
};
|
||||
let result = match timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, ack_attempt).await {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
Err(_) => {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
Err(Error::other("scoped dirty usage acknowledgement deadline elapsed"))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let activity = self.scanner_scoped_dirty_usage_activity_confirmation().await?;
|
||||
if activity.instance_id == instance_id {
|
||||
Ok(activity)
|
||||
} else {
|
||||
Err(Error::other(
|
||||
"scoped dirty usage acknowledgement peer restarted before activity confirmation",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if Self::is_network_like_error(&err) {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_scoped_dirty_usage_activity_confirmation(&self) -> Result<ScannerPeerActivity> {
|
||||
timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, self.scanner_activity())
|
||||
.await
|
||||
.map_err(|_| Error::other("scoped dirty usage activity confirmation timed out"))?
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
@@ -2836,16 +3036,79 @@ mod tests {
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
|
||||
},
|
||||
],
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
owner_id: "33333333-3333-3333-3333-333333333333".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_payloads_split_at_protocol_limit() {
|
||||
use rustfs_protos::scoped_dirty_usage::{SCOPED_DIRTY_USAGE_MAX_ENTRIES, canonical_scoped_dirty_usage_request};
|
||||
|
||||
let entries = (0..=SCOPED_DIRTY_USAGE_MAX_ENTRIES)
|
||||
.map(|index| ScannerScopedDirtyUsageAckEntry {
|
||||
bucket: format!("bucket-{index:02}"),
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: 9,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(
|
||||
"33333333-3333-3333-3333-333333333333".to_string(),
|
||||
"0123456789abcdef0123456789abcdef".to_string(),
|
||||
false,
|
||||
entries,
|
||||
)
|
||||
.expect("33 entries should split into valid scoped dirty usage requests");
|
||||
|
||||
assert_eq!(payloads.len(), 2);
|
||||
assert_eq!(payloads[0].entries.len(), SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
assert_eq!(payloads[1].entries.len(), 1);
|
||||
assert_eq!(payloads[0].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-00"));
|
||||
assert_eq!(payloads[0].entries.last().map(|entry| entry.bucket.as_str()), Some("bucket-31"));
|
||||
assert_eq!(payloads[1].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-32"));
|
||||
for payload in payloads {
|
||||
canonical_scoped_dirty_usage_request(&payload).expect("each split scoped ACK payload should be canonical");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
|
||||
let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
|
||||
@@ -2854,9 +3117,18 @@ mod tests {
|
||||
assert_eq!(decoded.generation, 7);
|
||||
assert_eq!(decoded.pending_bucket_count, 2);
|
||||
assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
|
||||
assert_eq!(decoded.owner_id, "33333333-3333-3333-3333-333333333333");
|
||||
assert!(decoded.complete);
|
||||
assert_eq!(decoded.buckets.get("archive"), Some(&3));
|
||||
assert_eq!(decoded.buckets.get("photos"), Some(&7));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("archive").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x11111111111111111111111111111111))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("archive").map(|bucket| bucket.generation), Some(3));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("photos").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x22222222222222222222222222222222))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("photos").map(|bucket| bucket.generation), Some(7));
|
||||
|
||||
let overflow_count =
|
||||
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
|
||||
@@ -2907,6 +3179,14 @@ mod tests {
|
||||
empty_bucket.buckets[0].bucket.clear();
|
||||
cases.push((empty_bucket, "empty bucket name"));
|
||||
|
||||
let mut invalid_owner = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_owner.owner_id.clear();
|
||||
cases.push((invalid_owner, "snapshot owner"));
|
||||
|
||||
let mut invalid_incarnation = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_incarnation.buckets[0].bucket_incarnation = Uuid::nil().as_bytes().to_vec().into();
|
||||
cases.push((invalid_incarnation, "bucket incarnation"));
|
||||
|
||||
let mut partial = test_scanner_dirty_usage_snapshot_response();
|
||||
partial.complete = false;
|
||||
cases.push((partial, "entry-limit overflow"));
|
||||
@@ -2919,6 +3199,7 @@ mod tests {
|
||||
.map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: format!("bucket-{index:04}"),
|
||||
generation: 1,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
})
|
||||
.collect(),
|
||||
..test_scanner_dirty_usage_snapshot_response()
|
||||
|
||||
@@ -449,14 +449,6 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), false, Some(max_bytes)).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -465,6 +457,14 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), false, Some(max_bytes)).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
@@ -476,6 +476,18 @@ where
|
||||
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
/// Read an existing config object without treating an empty payload as absent.
|
||||
/// Callers that validate their own payload format need to distinguish corruption
|
||||
/// from `ConfigNotFound`.
|
||||
|
||||
@@ -3385,7 +3385,7 @@ pub(crate) async fn acquire_pool_activation_fleet_proof(
|
||||
.ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
|
||||
}
|
||||
|
||||
pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
// Save-stage helpers add context by formatting the original error, so the
|
||||
// marker may be nested in the display string. Restrict matching to the
|
||||
// `Error::other` I/O shape used by this activation path.
|
||||
@@ -5108,49 +5108,7 @@ async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<Po
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
|
||||
let batch = uuid::Uuid::new_v4();
|
||||
for (pool, read) in reads.iter().enumerate() {
|
||||
let mut observation = serde_json::json!({
|
||||
"kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
|
||||
"cas": match &read.cas {
|
||||
PoolMetaCasToken::Missing => "missing",
|
||||
PoolMetaCasToken::Existing(_) => "existing",
|
||||
PoolMetaCasToken::Unsafe => "unsafe",
|
||||
},
|
||||
"etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
|
||||
});
|
||||
match &read.replica {
|
||||
PoolMetaReplica::Valid {
|
||||
raw,
|
||||
canonical,
|
||||
meta,
|
||||
revision,
|
||||
committed,
|
||||
..
|
||||
} => {
|
||||
observation["state"] = serde_json::json!("valid");
|
||||
observation["committed"] = serde_json::json!(committed);
|
||||
observation["version"] = serde_json::json!(revision.version);
|
||||
observation["cluster_id"] = serde_json::json!(revision.cluster_id);
|
||||
observation["epoch"] = serde_json::json!(revision.epoch);
|
||||
observation["generation"] = serde_json::json!(revision.generation);
|
||||
observation["transaction_id"] = serde_json::json!(revision.transaction_id);
|
||||
observation["pool_count"] = serde_json::json!(meta.pools.len());
|
||||
observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
|
||||
observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
|
||||
}
|
||||
PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
|
||||
PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
|
||||
PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
|
||||
PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
|
||||
}
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
}
|
||||
reads
|
||||
join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await
|
||||
}
|
||||
|
||||
fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
|
||||
@@ -5522,60 +5480,6 @@ fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
struct StartupCasObservation {
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: Vec<usize>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
tokio::task_local! {
|
||||
static STARTUP_CAS_OBSERVATION: StartupCasObservation;
|
||||
}
|
||||
|
||||
// This scope follows only the directly polled startup future. Spawned work
|
||||
// does not inherit it; receiver evidence retains its existing RPC tuple.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: &[Arc<S>],
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
STARTUP_CAS_OBSERVATION
|
||||
.scope(
|
||||
StartupCasObservation {
|
||||
attempt,
|
||||
phase,
|
||||
// These identities are never dereferenced or logged. The
|
||||
// caller and operation keep the same pool Arcs alive.
|
||||
pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
// Direct JSON diagnostics are independent of the startup tracing subscriber.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
|
||||
let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
|
||||
.ok()
|
||||
.and_then(|value| uuid::Uuid::parse_str(&value).ok())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
observation["nonce"] = serde_json::json!(nonce);
|
||||
observation["pid"] = serde_json::json!(std::process::id());
|
||||
let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
observation["attempt"] = serde_json::json!(scope.attempt);
|
||||
observation["startup_phase"] = serde_json::json!(scope.phase);
|
||||
});
|
||||
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
|
||||
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
|
||||
}
|
||||
|
||||
async fn save_pool_meta_object_cas<S>(
|
||||
pool: Arc<S>,
|
||||
object: &str,
|
||||
@@ -5596,43 +5500,13 @@ where
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_options(&mut opts);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
|
||||
serde_json::json!({
|
||||
"kind": "cas", "object": object, "phase": phase,
|
||||
"pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
|
||||
}).ok().flatten(),
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
|
||||
"if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
|
||||
"if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
|
||||
"tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
|
||||
"no_lock": opts.no_lock,
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let result = result.and_then(|object_info| {
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
});
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(mut observation) = observation {
|
||||
observation["ok"] = serde_json::json!(result.is_ok());
|
||||
observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
|
||||
observation["mod_time"] = serde_json::json!(
|
||||
result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|info| info.mod_time)
|
||||
.map(|time| time.unix_timestamp_nanos().to_string())
|
||||
);
|
||||
observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
result
|
||||
let object_info = result?;
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
}
|
||||
|
||||
async fn persist_pool_meta_identity<S>(
|
||||
@@ -6931,13 +6805,6 @@ impl PoolMeta {
|
||||
};
|
||||
if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
|
||||
persist_pool_meta_identity(pools, write_state, true, fence).await?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "confirmed", "object": POOL_META_NAME,
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
|
||||
"generation": confirmed.revision.generation,
|
||||
"transaction_id": confirmed.revision.transaction_id,
|
||||
}));
|
||||
return Ok(confirmed.meta);
|
||||
}
|
||||
if !commit_succeeded {
|
||||
|
||||
@@ -2673,7 +2673,7 @@ mod tests {
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(should_use_multipart_data_movement(&object_info, false));
|
||||
|
||||
let single_nonstandard_part = ObjectInfo {
|
||||
@@ -3050,7 +3050,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(object_info.parts.iter().any(|part| part.checksums.is_some()));
|
||||
let opts = data_movement_put_object_opts(&object_info, 0);
|
||||
assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_PART_CHECKSUMS));
|
||||
|
||||
@@ -324,6 +324,46 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
Box::pin(
|
||||
self.disk
|
||||
.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+458
-108
@@ -191,11 +191,33 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r
|
||||
}
|
||||
|
||||
async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> {
|
||||
write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await
|
||||
}
|
||||
|
||||
async fn write_delete_rollback_file(
|
||||
object_dir: &Path,
|
||||
rollback_dir: Uuid,
|
||||
name: &str,
|
||||
data: &[u8],
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let backup_dir = object_dir.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&backup_dir).await.map_err(to_file_error)?;
|
||||
fs::write(backup_dir.join(STORAGE_FORMAT_FILE_BACKUP), data)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
let path = backup_dir.join(name);
|
||||
if namespace_owner.is_none() {
|
||||
fs::create_dir_all(&backup_dir).await.map_err(to_file_error)?;
|
||||
fs::write(path, data).await.map_err(to_file_error)?;
|
||||
return Ok(());
|
||||
}
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
let data = data.to_vec();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
std::fs::create_dir_all(&backup_dir)?;
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
std::fs::write(path, data)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -342,6 +364,7 @@ struct DeleteVersionMutation {
|
||||
struct DeleteRollbackFailure {
|
||||
stage: &'static str,
|
||||
error: DiskError,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
async fn restore_delete_rollback_after_error(
|
||||
@@ -353,12 +376,18 @@ async fn restore_delete_rollback_after_error(
|
||||
failure: DeleteRollbackFailure,
|
||||
publication_root: &os::PublicationRoot,
|
||||
) -> DiskError {
|
||||
let DeleteRollbackFailure { stage, error } = failure;
|
||||
let DeleteRollbackFailure {
|
||||
stage,
|
||||
error,
|
||||
namespace_owner,
|
||||
} = failure;
|
||||
let Some(rollback_dir) = rollback_dir else {
|
||||
return error;
|
||||
};
|
||||
|
||||
if let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, publication_root).await {
|
||||
if let Err(restore_err) =
|
||||
restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, namespace_owner).await
|
||||
{
|
||||
warn!(
|
||||
volume,
|
||||
path,
|
||||
@@ -5654,6 +5683,123 @@ impl LocalDisk {
|
||||
// })
|
||||
// }
|
||||
|
||||
#[tracing::instrument(name = "delete_version", level = "trace", skip_all)]
|
||||
pub(in crate::disk) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.delete_version_inner(
|
||||
volume,
|
||||
path,
|
||||
fi,
|
||||
DeleteVersionMutation {
|
||||
force_del_marker,
|
||||
opts,
|
||||
namespace_owner,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "write_metadata", level = "trace", skip_all)]
|
||||
async fn write_metadata_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
crate::hp_guard!("LocalDisk::write_metadata");
|
||||
fi.validate_for_metadata_read()?;
|
||||
let p = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
if !fi.fresh {
|
||||
let (buf, _) = read_file_exists(&p).await?;
|
||||
if !buf.is_empty() {
|
||||
let _ = meta.unmarshal_msg(&buf).map_err(|_| {
|
||||
meta = FileMeta::new();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
meta.add_version(fi)?;
|
||||
|
||||
let fm_data = meta.marshal_msg()?;
|
||||
|
||||
// Atomic temp+rename: this path also rewrites live xl.meta (delete markers,
|
||||
// decommission), where an in-place truncate would expose torn metadata.
|
||||
self.write_all_meta_with_namespace_owner(
|
||||
volume,
|
||||
format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
|
||||
&fm_data,
|
||||
true,
|
||||
namespace_owner,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_data_dir_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DataDirDeleteStatus> {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
};
|
||||
{
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
if let Some(entry) = registry.entries.get_mut(&key) {
|
||||
if !entry.tokens.is_empty() {
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
return Ok(DataDirDeleteStatus::Deferred);
|
||||
}
|
||||
if entry.deleting {
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
return Ok(DataDirDeleteStatus::Deferred);
|
||||
}
|
||||
entry.deleting = true;
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
} else {
|
||||
registry.entries.insert(
|
||||
key.clone(),
|
||||
SnapshotLeaseEntry {
|
||||
pending_delete: Some(opts.clone()),
|
||||
deleting: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let result = self
|
||||
.delete_unleased_with_namespace_owner(volume, path, &opts, namespace_owner)
|
||||
.await;
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
registry.entries.remove(&key);
|
||||
Ok(DataDirDeleteStatus::Deleted)
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(entry) = registry.entries.get_mut(&key) {
|
||||
entry.deleting = false;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_version_inner(&self, volume: &str, path: &str, fi: FileInfo, mutation: DeleteVersionMutation) -> Result<()> {
|
||||
let DeleteVersionMutation {
|
||||
force_del_marker,
|
||||
@@ -5696,7 +5842,7 @@ impl LocalDisk {
|
||||
|
||||
if fi.deleted && force_del_marker {
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
|
||||
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), rollback_dir, namespace_owner.clone())
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -5712,7 +5858,14 @@ impl LocalDisk {
|
||||
let old_dir = meta.delete_version(&fi)?;
|
||||
let mut reserved_version_delete = false;
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
write_metadata_rollback_backup(file_path.as_path(), rollback_dir, &buf).await?;
|
||||
write_delete_rollback_file(
|
||||
file_path.as_path(),
|
||||
rollback_dir,
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
&buf,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if let Some(uuid) = old_dir {
|
||||
@@ -5728,6 +5881,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_metadata_update",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5745,6 +5899,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_data_path",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5753,7 +5908,7 @@ impl LocalDisk {
|
||||
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = file_path.join(rollback_dir.to_string());
|
||||
if let Err(err) = fs::create_dir_all(&rollback_path).await {
|
||||
if let Err(err) = os::create_dir_all_with_namespace_owner(&rollback_path, namespace_owner.clone()).await {
|
||||
let err: DiskError = to_file_error(err).into();
|
||||
return Err(restore_delete_rollback_after_error(
|
||||
file_path.as_path(),
|
||||
@@ -5764,12 +5919,16 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_rollback_dir",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
reserved_version_delete = match self.reserve_version_delete(volume, path, uuid, rollback_dir).await {
|
||||
reserved_version_delete = match self
|
||||
.reserve_version_delete_with_namespace_owner(volume, path, uuid, rollback_dir, namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
Ok(reserved) => reserved,
|
||||
Err(err) => {
|
||||
return Err(restore_delete_rollback_after_error(
|
||||
@@ -5781,6 +5940,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_reserve_data",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5789,9 +5949,14 @@ impl LocalDisk {
|
||||
};
|
||||
let rollback_data_path = rollback_path.join(uuid.to_string());
|
||||
if !reserved_version_delete
|
||||
&& let Err(err) =
|
||||
rename_all_ignore_missing_source(&old_path, &rollback_data_path, &rollback_path, &self.publication_root)
|
||||
.await
|
||||
&& let Err(err) = os::rename_all_ignore_missing_source_with_owner(
|
||||
&old_path,
|
||||
&rollback_data_path,
|
||||
&rollback_path,
|
||||
&self.publication_root,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err(restore_delete_rollback_after_error(
|
||||
file_path.as_path(),
|
||||
@@ -5802,6 +5967,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_stage_data",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5810,13 +5976,16 @@ impl LocalDisk {
|
||||
if should_fail_after_delete_data_staged(path) {
|
||||
if reserved_version_delete {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete(
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
file_path.as_path(),
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
"delete_version_test_after_stage",
|
||||
DiskError::Unexpected,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_test_after_stage",
|
||||
error: DiskError::Unexpected,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -5829,6 +5998,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_test_after_stage",
|
||||
error: DiskError::Unexpected,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5858,13 +6028,16 @@ impl LocalDisk {
|
||||
let err: DiskError = err.into();
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete(
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
file_path.as_path(),
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
"delete_version_metadata_encode",
|
||||
err,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_metadata_encode",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -5877,6 +6050,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_metadata_encode",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5899,7 +6073,17 @@ impl LocalDisk {
|
||||
if let Err(err) = commit_result {
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete(file_path.as_path(), rollback_dir, volume, path, "delete_version_commit", err)
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
file_path.as_path(),
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_commit",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.await);
|
||||
}
|
||||
return Err(restore_delete_rollback_after_error(
|
||||
@@ -5911,6 +6095,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_commit",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -5919,16 +6104,21 @@ impl LocalDisk {
|
||||
|
||||
if reserved_version_delete
|
||||
&& let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
|
||||
&& let Err(err) = self
|
||||
.commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
return Err(self
|
||||
.abort_reserved_version_delete(
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
file_path.as_path(),
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
"delete_version_commit_intent",
|
||||
err,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_version_commit_intent",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6098,7 +6288,7 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "delete", level = "trace", skip_all)]
|
||||
async fn delete_with_namespace_owner(
|
||||
pub(in crate::disk) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
@@ -6111,7 +6301,8 @@ impl LocalDisk {
|
||||
&& let Some((object, transaction_id)) = path.rsplit_once('/')
|
||||
&& let Ok(transaction_id) = Uuid::parse_str(transaction_id)
|
||||
{
|
||||
self.finish_version_delete(volume, object, transaction_id).await?
|
||||
self.finish_version_delete(volume, object, transaction_id, namespace_owner.clone())
|
||||
.await?
|
||||
} else {
|
||||
false
|
||||
};
|
||||
@@ -6453,19 +6644,27 @@ impl LocalDisk {
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
rollback_dir: Option<Uuid>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
|
||||
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
write_delete_rollback_file(object_dir, rollback_dir, DELETE_MARKER_ROLLBACK_FILE, &[], namespace_owner.clone())
|
||||
.await?;
|
||||
}
|
||||
if let Err(err) = self.write_metadata("", volume, path, fi).await {
|
||||
if let Err(err) = self
|
||||
.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
|
||||
&& let Err(restore_err) = restore_delete_rollback_with_namespace_owner(
|
||||
object_dir,
|
||||
&xl_path,
|
||||
rollback_dir,
|
||||
&self.publication_root,
|
||||
namespace_owner,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
|
||||
@@ -6510,7 +6709,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
};
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
|
||||
.write_missing_delete_marker(volume, path, delete_marker, object_dir, opts.old_data_dir, None)
|
||||
.await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
@@ -6559,6 +6758,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_update",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6594,6 +6794,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_data_path",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6625,6 +6826,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_rollback_dir",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6665,6 +6867,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_stage_data",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6692,6 +6895,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_test_after_stage",
|
||||
error: DiskError::Unexpected,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6734,6 +6938,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_delete",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6780,6 +6985,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_encode",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6805,6 +7011,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_write",
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -8015,6 +8222,18 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
|
||||
self.reserve_version_delete_with_namespace_owner(volume, object, data_dir, rollback_dir, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_version_delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
data_dir: Uuid,
|
||||
rollback_dir: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
let path = format!("{object}/{data_dir}");
|
||||
let data_path = self.io_get_object_path(volume, &path)?;
|
||||
match fs::metadata(&data_path).await {
|
||||
@@ -8024,6 +8243,28 @@ impl LocalDisk {
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
let marker_path = data_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"));
|
||||
if namespace_owner.is_some() {
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker_path, namespace_owner.clone()).await;
|
||||
let volume = volume.to_string();
|
||||
let sync = os::run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&marker_path);
|
||||
let marker = std::fs::File::create(marker_path)?;
|
||||
let sync = effective_durability(&volume).syncs_commit_metadata();
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
}
|
||||
Ok(sync)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
if sync {
|
||||
os::fsync_dir_with_owner(&data_path, namespace_owner)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
let marker = File::create(marker_path).await.map_err(to_file_error)?;
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
marker.sync_all().await.map_err(to_file_error)?;
|
||||
@@ -8033,6 +8274,17 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> {
|
||||
self.commit_reserved_version_delete_with_namespace_owner(volume, object, rollback_dir, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn commit_reserved_version_delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
rollback_dir: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let object_path = self.io_get_object_path(volume, object)?;
|
||||
let mut entries = match fs::read_dir(object_path).await {
|
||||
Ok(entries) => entries,
|
||||
@@ -8048,10 +8300,14 @@ impl LocalDisk {
|
||||
continue;
|
||||
}
|
||||
let reserved_path = entry.path().join(&reserved_name);
|
||||
match fs::rename(&reserved_path, entry.path().join(&committed_name)).await {
|
||||
match os::rename_with_namespace_owner(&reserved_path, &entry.path().join(&committed_name), namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
os::fsync_dir(&entry.path()).await.map_err(to_file_error)?;
|
||||
os::fsync_dir_with_owner(&entry.path(), namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
@@ -8061,7 +8317,13 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finish_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<bool> {
|
||||
async fn finish_version_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
rollback_dir: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
let object_path = self.io_get_object_path(volume, object)?;
|
||||
let mut entries = match fs::read_dir(object_path).await {
|
||||
Ok(entries) => entries,
|
||||
@@ -8082,13 +8344,14 @@ impl LocalDisk {
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
if let Err(err) = self
|
||||
.delete_data_dir(
|
||||
.delete_data_dir_with_namespace_owner(
|
||||
volume,
|
||||
&format!("{object}/{data_dir}"),
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await
|
||||
&& first_err.is_none()
|
||||
@@ -8109,6 +8372,28 @@ impl LocalDisk {
|
||||
object: &str,
|
||||
stage: &'static str,
|
||||
err: DiskError,
|
||||
) -> DiskError {
|
||||
self.abort_reserved_version_delete_with_failure(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
object,
|
||||
DeleteRollbackFailure {
|
||||
stage,
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn abort_reserved_version_delete_with_failure(
|
||||
&self,
|
||||
object_dir: &Path,
|
||||
rollback_dir: Uuid,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
failure: DeleteRollbackFailure,
|
||||
) -> DiskError {
|
||||
let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
restore_delete_rollback_after_error(
|
||||
@@ -8117,7 +8402,7 @@ impl LocalDisk {
|
||||
Some(rollback_dir),
|
||||
volume,
|
||||
object,
|
||||
DeleteRollbackFailure { stage, error: err },
|
||||
failure,
|
||||
&self.publication_root,
|
||||
)
|
||||
.await
|
||||
@@ -9608,49 +9893,7 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
};
|
||||
{
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
if let Some(entry) = registry.entries.get_mut(&key) {
|
||||
if !entry.tokens.is_empty() {
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
return Ok(DataDirDeleteStatus::Deferred);
|
||||
}
|
||||
if entry.deleting {
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
return Ok(DataDirDeleteStatus::Deferred);
|
||||
}
|
||||
entry.deleting = true;
|
||||
entry.pending_delete.get_or_insert_with(|| opts.clone());
|
||||
} else {
|
||||
registry.entries.insert(
|
||||
key.clone(),
|
||||
SnapshotLeaseEntry {
|
||||
pending_delete: Some(opts.clone()),
|
||||
deleting: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let result = self.delete_unleased(volume, path, &opts).await;
|
||||
let mut registry = self.snapshot_leases.lock().await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
registry.entries.remove(&key);
|
||||
Ok(DataDirDeleteStatus::Deleted)
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(entry) = registry.entries.get_mut(&key) {
|
||||
entry.deleting = false;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
self.delete_data_dir_with_namespace_owner(volume, path, opts, None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -9689,32 +9932,8 @@ impl DiskAPI for LocalDisk {
|
||||
Err(Error::other("Invalid Argument"))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
crate::hp_guard!("LocalDisk::write_metadata");
|
||||
fi.validate_for_metadata_read()?;
|
||||
let p = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
if !fi.fresh {
|
||||
let (buf, _) = read_file_exists(&p).await?;
|
||||
if !buf.is_empty() {
|
||||
let _ = meta.unmarshal_msg(&buf).map_err(|_| {
|
||||
meta = FileMeta::new();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
meta.add_version(fi)?;
|
||||
|
||||
let fm_data = meta.marshal_msg()?;
|
||||
|
||||
// Atomic temp+rename: this path also rewrites live xl.meta (delete markers,
|
||||
// decommission), where an in-place truncate would expose torn metadata.
|
||||
self.write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &fm_data, true)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
self.write_metadata_with_namespace_owner(volume, path, fi, None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -22177,4 +22396,135 @@ mod test {
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn single_delete_internal_restore_keeps_owner_after_cancellation() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
use futures::FutureExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("fixture directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let bucket = "single-delete-internal-restore";
|
||||
let object = format!("object-{}", Uuid::new_v4());
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
let version = Uuid::new_v4();
|
||||
let data_dir = Uuid::new_v4();
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
let fi = test_file_info(&object, version, Some(data_dir), None);
|
||||
let original = test_meta(fi.clone());
|
||||
let object_dir = disk.io_get_object_path(bucket, &object).expect("object IO path");
|
||||
let part = object_dir.join(data_dir.to_string()).join("part.1");
|
||||
let metadata = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let backup = object_dir.join(rollback_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(part.parent().expect("data parent"))
|
||||
.await
|
||||
.expect("data directory");
|
||||
fs::write(&part, b"x").await.expect("real shard");
|
||||
fs::write(&metadata, &original).await.expect("real version metadata");
|
||||
set_delete_version_fail_after_data_staged(&object);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = hooks::install_at(hooks::Stage::Rename, &metadata, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let before = ctx.namespace_commit_generation();
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let deleting_disk = Arc::clone(&disk);
|
||||
let deleting_object = object.clone();
|
||||
let mut delete = tokio::spawn(async move {
|
||||
deleting_disk
|
||||
.delete_version_inner(
|
||||
bucket,
|
||||
&deleting_object,
|
||||
fi,
|
||||
DeleteVersionMutation {
|
||||
force_del_marker: false,
|
||||
opts: DeleteOptions {
|
||||
old_data_dir: Some(rollback_dir),
|
||||
..Default::default()
|
||||
},
|
||||
namespace_owner: Some(owner),
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
let mut joined = false;
|
||||
let mut entered = false;
|
||||
let mut counts = None;
|
||||
let observations = std::panic::AssertUnwindSafe(async {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = entered_rx => {
|
||||
result.expect("actual internal restore entry");
|
||||
entered = true;
|
||||
}
|
||||
result = &mut delete => {
|
||||
joined = true;
|
||||
panic!("delete returned before internal physical restore: {result:?}");
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("internal restore must reach the physical rename");
|
||||
assert_eq!(std::fs::read(&backup).expect("real undo backup"), original);
|
||||
assert_eq!(std::fs::read(&part).expect("reserved shard"), b"x");
|
||||
delete.abort();
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), &mut delete).await;
|
||||
joined = result.is_ok();
|
||||
assert!(
|
||||
result
|
||||
.expect("cancelled caller joins")
|
||||
.expect_err("cancelled caller")
|
||||
.is_cancelled()
|
||||
);
|
||||
counts = Some((ctx.namespace_commits_pending(), ctx.namespace_commit_generation()));
|
||||
assert!(hooks::drain_namespace_key(&metadata).now_or_never().is_none());
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
|
||||
drop(release);
|
||||
drop(hook);
|
||||
let coordinator_drained = joined || tokio::time::timeout(Duration::from_secs(10), &mut delete).await.is_ok();
|
||||
if !coordinator_drained {
|
||||
delete.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), &mut delete).await;
|
||||
}
|
||||
let physical_drained = tokio::time::timeout(Duration::from_secs(5), hooks::drain_namespace_key(&metadata))
|
||||
.await
|
||||
.is_ok();
|
||||
let owner_drained = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
if !entered || !coordinator_drained || !physical_drained || !owner_drained {
|
||||
eprintln!("internal restore cleanup incomplete; retained root: {:?}", dir.keep());
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
panic!("internal restore cleanup must drain before removing its root");
|
||||
}
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
assert_eq!(std::fs::read(&metadata).expect("late restored metadata"), original);
|
||||
assert!(!backup.exists(), "the actual backup rename must have completed");
|
||||
assert_eq!(std::fs::read(&part).expect("old shard survives"), b"x");
|
||||
disk.read_version("", bucket, &object, &version.to_string(), &ReadOptions::default())
|
||||
.await
|
||||
.expect("restored version");
|
||||
let (pending, generation) = counts.expect("observations completed");
|
||||
assert!(pending, "internal error recovery lost the physical namespace owner");
|
||||
assert_eq!(generation, before + 1);
|
||||
assert_eq!(ctx.namespace_commit_generation(), before + 2);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -732,6 +732,46 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete_version(volume, path, fi, force_del_marker, opts).await;
|
||||
// This is sender lifetime only, not proof of a remote physical drain.
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.delete_with_namespace_owner(volume, path, opts, namespace_owner).await,
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete(volume, path, opts).await;
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep local undo publication owned independently of the wrapper deadline.
|
||||
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
|
||||
pub(crate) async fn undo_write_with_namespace_owner(
|
||||
|
||||
@@ -247,7 +247,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
#[cfg(all(test, not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
|
||||
@@ -256,9 +256,7 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
}
|
||||
|
||||
@@ -274,7 +272,6 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
install_at(Stage::PreparedRename, path, hook)
|
||||
}
|
||||
@@ -291,50 +288,45 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Controlled application-test pause at an existing physical executor boundary.
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub struct LocalPublicationPause {
|
||||
_hook: prepared_publication_test_hooks::Guard,
|
||||
entered: oneshot::Receiver<()>,
|
||||
_release: std::sync::mpsc::Sender<()>,
|
||||
}
|
||||
#[cfg(test)]
|
||||
type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
|
||||
#[cfg(test)]
|
||||
static RENAME_DESTINATIONS: LazyLock<Mutex<HashMap<PathBuf, RenameDestinationHook>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RenameDestinationGuard(PathBuf);
|
||||
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
impl LocalPublicationPause {
|
||||
pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result<Self> {
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local(volume, path)
|
||||
.ok_or(DiskError::DiskNotFound)??;
|
||||
let stage = match stage {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
};
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
Ok(Self {
|
||||
_hook: hook,
|
||||
entered,
|
||||
_release: release,
|
||||
})
|
||||
#[cfg(test)]
|
||||
impl Drop for RenameDestinationGuard {
|
||||
fn drop(&mut self) {
|
||||
RENAME_DESTINATIONS.lock().remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
(&mut self.entered).await
|
||||
#[cfg(test)]
|
||||
pub(crate) fn observe_rename_destination(source: &Path, hook: impl FnOnce(&Path) + Send + 'static) -> RenameDestinationGuard {
|
||||
assert!(
|
||||
RENAME_DESTINATIONS
|
||||
.lock()
|
||||
.insert(source.to_path_buf(), Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
RenameDestinationGuard(source.to_path_buf())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn drain_namespace_key(path: &Path) {
|
||||
drop(super::acquire_namespace_mutation_lease(path).await);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn run_rename_destination(source: &Path, destination: &Path) {
|
||||
let hook = RENAME_DESTINATIONS.lock().remove(source);
|
||||
if let Some(hook) = hook {
|
||||
hook(destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1409,7 +1401,7 @@ async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationL
|
||||
acquire_namespace_mutation_lease_with_owner(path, None).await
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease_with_owner(
|
||||
pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
@@ -1964,7 +1956,7 @@ pub(crate) async fn remove_file_with_owner(
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
std::fs::remove_file(path)
|
||||
})
|
||||
@@ -1984,6 +1976,42 @@ pub(crate) async fn remove_dir_with_owner(
|
||||
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
|
||||
/// Preserve raw rename semantics while retaining a counted owner in the syscall.
|
||||
/// Unlike reliable rename, this never creates parents or retries a missing source.
|
||||
pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
src: &Path,
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
let src = src.to_path_buf();
|
||||
let dst = dst.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst);
|
||||
}
|
||||
std::fs::rename(src, dst)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
@@ -2188,7 +2216,7 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
move || {
|
||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
@@ -2319,7 +2347,9 @@ async fn reliable_rename_inner_with_lease(
|
||||
let base_dir = base_dir.clone();
|
||||
move || {
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run_rename_destination(&src_file_path, &dst_file_path);
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
|
||||
@@ -2278,6 +2278,121 @@ mod tests {
|
||||
assert_eq!(read, fixture.plaintext, "SSE-C + compression full GET must reassemble all parts");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_reads_preserve_plaintext() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 0];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
for (kind, mut fixture, headers) in [
|
||||
(
|
||||
"encrypted",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"compressed and encrypted",
|
||||
compressed_encrypted_multipart_fixture(key, &part_sizes).await,
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
fixture.object_info.etag = Some(faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref()));
|
||||
assert_eq!(fixture.object_info.etag.as_ref().expect("source ETag").len(), 32);
|
||||
assert_eq!(fixture.object_info.parts.len(), 2);
|
||||
let tail = &fixture.object_info.parts[1];
|
||||
assert_eq!(tail.actual_size, 0, "{kind}: final part has no plaintext");
|
||||
if kind == "compressed" {
|
||||
assert_eq!(tail.size, 0, "unpadded compression emits no bytes for an empty part");
|
||||
} else {
|
||||
assert!(tail.size > 0, "{kind}: the empty part still has a stored frame");
|
||||
}
|
||||
let stored_size = i64::try_from(fixture.stored.len()).expect("fixture size fits i64");
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored)),
|
||||
None,
|
||||
&fixture.object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("full transformed read must include the empty tail");
|
||||
assert_eq!((offset, length), (0, stored_size), "{kind}: full read includes all stored parts");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read through the complete decoder EOF");
|
||||
assert_eq!(body, fixture.plaintext, "{kind}: no plaintext is added or lost by the empty tail");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_read_authenticates_v2_final_frame() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let plaintext = legacy_fixture_part_plaintext(1, 5 * 1024 * 1024);
|
||||
let mut ciphertext = Vec::new();
|
||||
let mut parts = Vec::new();
|
||||
for (number, body) in [(1, plaintext.as_slice()), (2, b"".as_slice())] {
|
||||
let start = ciphertext.len();
|
||||
rustfs_rio::EncryptReader::new_multipart_v2(Cursor::new(body), key, LEGACY_FIXTURE_BASE_NONCE, number)
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("encrypt a v2 fixture part with an authenticated final frame");
|
||||
parts.push(ObjectPartInfo {
|
||||
number,
|
||||
size: ciphertext.len() - start,
|
||||
actual_size: i64::try_from(body.len()).expect("fixture plaintext size fits"),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let tail_start = parts[0].size;
|
||||
assert_eq!(parts[1].actual_size, 0);
|
||||
assert!(parts[1].size > 8, "the empty final frame carries more than an END marker");
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "v2-empty-tail".to_string(),
|
||||
size: i64::try_from(ciphertext.len()).expect("fixture ciphertext size fits"),
|
||||
etag: Some(faster_hex::hex_string(Md5::digest(&plaintext).as_ref())),
|
||||
parts: Arc::new(parts),
|
||||
user_defined: Arc::new(legacy_ssec_multipart_metadata(key, plaintext.len())),
|
||||
..Default::default()
|
||||
};
|
||||
for corrupt_tail in [false, true] {
|
||||
let mut stored = ciphertext.clone();
|
||||
if corrupt_tail {
|
||||
// The v2 header is authenticated associated data, including
|
||||
// the header of a final frame containing zero plaintext.
|
||||
stored[tail_start + 5] ^= 1;
|
||||
}
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(stored)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&ssec_headers_from_key(key),
|
||||
)
|
||||
.await
|
||||
.expect("construct the full reader before consuming the final frame");
|
||||
assert_eq!((offset, length), (0, object_info.size));
|
||||
let result = tokio::io::copy(&mut reader.stream, &mut tokio::io::sink()).await;
|
||||
if corrupt_tail {
|
||||
let err = result.expect_err("EOF must authenticate the empty final frame after all plaintext is returned");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "v2 encrypted frame failed authentication");
|
||||
} else {
|
||||
assert_eq!(
|
||||
result.expect("valid empty final frame must reach EOF"),
|
||||
u64::try_from(plaintext.len()).expect("plaintext length fits")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compressed_encrypted_multipart_range_crosses_part_boundary() {
|
||||
let key_bytes = [0x6Eu8; 32];
|
||||
@@ -3656,6 +3771,61 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_full_read_preserves_legacy_zero_and_negative_part_sizes() {
|
||||
let key = [0x77; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 1024 * 1024];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
// The encrypted case supplies the fixture key explicitly. This covers
|
||||
// full decrypted reads, not managed-key acquisition.
|
||||
for (kind, fixture, headers) in [
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"encrypted with supplied key",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
let source_etag = faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref());
|
||||
assert_eq!(source_etag.len(), 32);
|
||||
assert_eq!(fixture.plaintext.len(), 6 * 1024 * 1024);
|
||||
for part_index in 0..part_sizes.len() {
|
||||
assert!(fixture.object_info.parts[part_index].actual_size > 0, "the selected part is nonempty");
|
||||
for actual_size in [0, -1] {
|
||||
let mut object_info = fixture.object_info.clone();
|
||||
object_info.etag = Some(source_etag.clone());
|
||||
Arc::make_mut(&mut object_info.parts)[part_index].actual_size = actual_size;
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored.clone())),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("the authoritative total size must keep full legacy reads available");
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, i64::try_from(fixture.stored.len()).expect("stored size fits"));
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("full read must reach EOF despite an unspecified per-part logical size");
|
||||
assert_eq!(
|
||||
body, fixture.plaintext,
|
||||
"{kind}: part {part_index} with actual_size={actual_size} must not lose readable data"
|
||||
);
|
||||
assert_eq!(reader.object_info.etag.as_deref(), Some(source_etag.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The physical part sizes must add up to `oi.size` for a seek to be safe;
|
||||
/// inconsistent metadata must fall back to the previous full-object read
|
||||
/// instead of scheduling an erasure read past the object end.
|
||||
|
||||
@@ -1597,7 +1597,7 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn is_multipart(&self) -> bool {
|
||||
self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
self.parts.len() > 1 || self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
}
|
||||
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
@@ -2235,6 +2235,35 @@ mod tests {
|
||||
}
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntry, TRANSITION_COMPLETE};
|
||||
|
||||
#[test]
|
||||
fn multipart_identity_uses_stored_parts_and_preserves_the_etag_fallback() {
|
||||
let plain_etag = "0123456789abcdef0123456789abcdef";
|
||||
let multipart_etag = "0123456789abcdef0123456789abcdef-1";
|
||||
for (case, part_count, etag, expected) in [
|
||||
("preserved source ETag", 2, Some(plain_etag), true),
|
||||
("missing ETag", 2, None, true),
|
||||
("ordinary PUT", 1, Some(plain_etag), false),
|
||||
("ordinary PUT without ETag", 1, None, false),
|
||||
("single-part MPU", 1, Some(multipart_etag), true),
|
||||
("legacy MPU without parts", 0, Some(multipart_etag), true),
|
||||
] {
|
||||
let object = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
parts: Arc::new(
|
||||
(1..=part_count)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(object.is_multipart(), expected, "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_fast_path_object(size: i64, versioned: bool) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
size,
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||
use crate::cluster::rpc::{
|
||||
PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
PeerRestClient, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot,
|
||||
ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
@@ -33,11 +34,11 @@ use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{
|
||||
Arc, Mutex, OnceLock,
|
||||
Arc, LazyLock, Mutex, OnceLock,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -311,12 +312,29 @@ pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one immutable ILM recovery export.
|
||||
pub struct IlmRecoveryExportFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for emitting the compact transition-transaction
|
||||
/// state sequence. The generation permit prevents a successor proof from
|
||||
/// being published until the admitted writer has finished.
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
@@ -338,6 +356,14 @@ fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::R
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
ILM_RECOVERY_EXPORT_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -438,6 +464,33 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat
|
||||
fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0)
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_transition_transaction_compaction_fleet_proof() -> Option<TransitionTransactionCompactionFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let token = acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now())?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(TransitionTransactionCompactionFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub(crate) fn transition_transaction_compaction_fleet_proof_matches(
|
||||
proof: &TransitionTransactionCompactionFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(&state, &proof.token, expected_topology, Instant::now())
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = cross_pool_fence_fleet_proof_slot()
|
||||
@@ -573,6 +626,117 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed = observe_ilm_recovery_export_fleet(expected_topology).await?;
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, &proof, expected_topology, &observed, Instant::now())
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_ilm_recovery_export_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(IlmRecoveryExportFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub async fn ilm_recovery_export_fleet_proof_matches(proof: &IlmRecoveryExportFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !ilm_recovery_export_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed) = observe_ilm_recovery_export_fleet(expected_topology).await else {
|
||||
return false;
|
||||
};
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, proof, expected_topology, &observed, Instant::now())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_topology_generation(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-topology-v1\0");
|
||||
hasher.update(proof.token.topology_fingerprint.as_bytes());
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_member_epochs_sha256(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let encoded = serde_json::to_vec(proof.token.peer_epochs.as_ref()).expect("member epoch map is JSON encodable");
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-members-v1\0");
|
||||
hasher.update(encoded);
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_local_process_epoch() -> Uuid {
|
||||
*ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
ilm_recovery_export_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed
|
||||
}
|
||||
|
||||
async fn observe_ilm_recovery_export_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, Instant::now()) {
|
||||
return state.proof.as_ref().map(|proof| proof.peer_epochs.as_ref().clone());
|
||||
}
|
||||
}
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
@@ -661,7 +825,7 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
state.proof.clone()
|
||||
} else {
|
||||
Some(FleetCapabilityProof::new(
|
||||
topology,
|
||||
topology.clone(),
|
||||
Arc::new(BTreeMap::new()),
|
||||
now + Duration::from_secs(60 * 60),
|
||||
))
|
||||
@@ -694,6 +858,21 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
decommission_state.topology_conflict = false;
|
||||
decommission_state.draining_generation = None;
|
||||
decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
drop(decommission_state);
|
||||
let mut export_state = ilm_recovery_export_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !fleet_capability_proof_valid_at(export_state.proof.as_ref(), &topology, now) {
|
||||
debug_assert!(
|
||||
export_state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.generation.is_drained())
|
||||
);
|
||||
export_state.topology_conflict = false;
|
||||
export_state.draining_generation = None;
|
||||
export_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -934,6 +1113,35 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
|
||||
RemoteVersionStateFleetProofGuard
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionTransactionCompactionFleetProofGuard {
|
||||
fn drop(&mut self) {
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn install_transition_transaction_compaction_fleet_proof_for_test(
|
||||
topology_fingerprint: &str,
|
||||
) -> TransitionTransactionCompactionFleetProofGuard {
|
||||
let _ = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.to_string());
|
||||
let effective_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
|
||||
.get()
|
||||
.expect("transition transaction compaction test topology should be initialized");
|
||||
if let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
effective_topology,
|
||||
Ok(BTreeMap::new()),
|
||||
Instant::now(),
|
||||
) {
|
||||
panic!("test proof installation must not fail: {err}");
|
||||
}
|
||||
TransitionTransactionCompactionFleetProofGuard
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
@@ -950,6 +1158,8 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -959,29 +1169,59 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
let notification_sys = get_global_notification_sys();
|
||||
let remote_version_state_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let fence_probe = match get_global_notification_sys() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
let cross_pool_fence_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let recovery_export_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
|
||||
None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let transition_transaction_compaction_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
|
||||
None => Err(Error::other(
|
||||
"transition transaction compaction fleet capability notification system is unavailable",
|
||||
)),
|
||||
}
|
||||
};
|
||||
let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
|
||||
remote_version_state_probe,
|
||||
cross_pool_fence_probe,
|
||||
recovery_export_probe,
|
||||
transition_transaction_compaction_probe
|
||||
);
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
@@ -1004,6 +1244,8 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1030,6 +1272,42 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
recovery_export_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "ilm_recovery_export_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
transition_transaction_compaction_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "transition_transaction_compaction_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
@@ -1152,6 +1430,28 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other(
|
||||
"transition transaction compaction capability fleet membership is incomplete",
|
||||
));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("transition transaction compaction capability peer is unreachable"))?;
|
||||
client
|
||||
.probe_transition_transaction_compaction(topology_fingerprint.to_string())
|
||||
.await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
|
||||
@@ -1174,6 +1474,46 @@ impl NotificationSys {
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
|
||||
async fn probe_ilm_recovery_export_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership is incomplete"));
|
||||
}
|
||||
let local_member = runtime_sources::local_node_name().await;
|
||||
if local_member.trim().is_empty() {
|
||||
return Err(Error::other("ILM recovery export local member identity is unavailable"));
|
||||
}
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, local_member.clone(), ilm_recovery_export_local_process_epoch())?;
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("ILM recovery export capability peer is unreachable"))?;
|
||||
client.probe_ilm_recovery_export(topology_fingerprint.to_string()).await
|
||||
});
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
validate_ilm_recovery_export_members(&self.peer_topology_hosts, &local_member, &peer_epochs)?;
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_ilm_recovery_export_members(
|
||||
expected_remote_members: &[String],
|
||||
local_member: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
) -> Result<()> {
|
||||
let expected = expected_remote_members
|
||||
.iter()
|
||||
.cloned()
|
||||
.chain(std::iter::once(local_member.to_string()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
if expected.len() != expected_remote_members.len().saturating_add(1) || observed.keys().ne(expected.iter()) {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership does not match topology"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rolling tier activity summed over every cluster member that answered, with
|
||||
@@ -2341,11 +2681,70 @@ impl NotificationSys {
|
||||
Ok(snapshots)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
|
||||
pub async fn scanner_scoped_dirty_usage_capabilities(
|
||||
&self,
|
||||
acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>,
|
||||
) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for (host, instance_id, generation) in acknowledgements {
|
||||
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
|
||||
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
ScannerDirtyUsageAcknowledgement::Generation { .. } => {
|
||||
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
|
||||
}
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
let clients = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|client| (client.grid_host.clone(), client.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
return Err(Error::other("scanner scoped dirty usage capability failed: peer is not reachable"));
|
||||
};
|
||||
futures.push(async move {
|
||||
let ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} = acknowledgement
|
||||
else {
|
||||
unreachable!("scoped acknowledgement was validated before probing");
|
||||
};
|
||||
timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
client.scanner_scoped_dirty_usage_capability(owner_id, instance_id, entries),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner scoped dirty usage capability timed out"))?
|
||||
});
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if !result? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
|
||||
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2357,18 +2756,34 @@ impl NotificationSys {
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, (instance_id, generation)) in by_host {
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
|
||||
continue;
|
||||
};
|
||||
futures.push(async move {
|
||||
let result = scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await;
|
||||
let result = match acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
instance_id, generation, ..
|
||||
} => {
|
||||
scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} => {
|
||||
client
|
||||
.acknowledge_scanner_scoped_dirty_usage(owner_id, instance_id, entries)
|
||||
.await
|
||||
}
|
||||
};
|
||||
(host, result)
|
||||
});
|
||||
}
|
||||
@@ -3507,6 +3922,81 @@ mod tests {
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_member_digest_is_order_independent_and_epoch_bound() {
|
||||
let now = Instant::now();
|
||||
let local_epoch = ilm_recovery_export_local_process_epoch();
|
||||
assert!(!local_epoch.is_nil());
|
||||
assert_eq!(local_epoch, ilm_recovery_export_local_process_epoch());
|
||||
let remote_epoch = Uuid::new_v4();
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let peers = BTreeMap::from([("node-b".to_string(), remote_epoch), ("node-a".to_string(), local_epoch)]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let proof = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
let digest = ilm_recovery_export_member_epochs_sha256(&proof);
|
||||
|
||||
let changed_slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let changed = BTreeMap::from([("node-a".to_string(), local_epoch), ("node-b".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&changed_slot, "topology-a", Ok(changed), now).is_none());
|
||||
let changed_proof = {
|
||||
let state = changed_slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
assert_ne!(digest, ilm_recovery_export_member_epochs_sha256(&changed_proof));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_members_must_match_the_exact_topology() {
|
||||
let expected_remote = vec!["node-b".to_string()];
|
||||
let local = "node-a";
|
||||
let complete = BTreeMap::from([
|
||||
(local.to_string(), Uuid::new_v4()),
|
||||
(expected_remote[0].clone(), Uuid::new_v4()),
|
||||
]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &complete).is_ok());
|
||||
|
||||
let unexpected = BTreeMap::from([(local.to_string(), Uuid::new_v4()), ("node-c".to_string(), Uuid::new_v4())]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &unexpected).is_err());
|
||||
assert!(
|
||||
validate_ilm_recovery_export_members(&[local.to_string()], local, &complete).is_err(),
|
||||
"the configured remote set cannot repeat the local member"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_restart_revokes_authority_until_permit_drains() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("fresh fleet should admit export")
|
||||
};
|
||||
|
||||
let restarted = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
let draining = publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted.clone()), now)
|
||||
.expect("restart must wait for the admitted export effect window");
|
||||
assert!(draining.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(!ilm_recovery_export_fleet_proof_matches_at(&state, &admitted, "topology-a", now));
|
||||
assert!(
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_none(),
|
||||
"successor authority must wait for the old effect window to drain"
|
||||
);
|
||||
}
|
||||
drop(admitted);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
|
||||
let topology = "topology-a";
|
||||
@@ -4458,15 +4948,28 @@ mod tests {
|
||||
peer_topology_hosts: Vec::new(),
|
||||
};
|
||||
let missing = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
|
||||
.acknowledge_scanner_dirty_usage(vec![ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
}])
|
||||
.await
|
||||
.expect_err("a missing acknowledgement target must remain pending");
|
||||
assert!(missing.to_string().contains("peer is not reachable"));
|
||||
|
||||
let duplicate = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
host: "peer-1".to_string(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
])
|
||||
.await
|
||||
.expect_err("duplicate acknowledgement targets must be rejected");
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_util;
|
||||
#[allow(clippy::module_inception, reason = "preserve the public services::tier::tier path")]
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use bytes::Bytes;
|
||||
@@ -802,7 +800,7 @@ pub enum TierConfigUpdateError {
|
||||
}
|
||||
|
||||
enum TierCandidateMutation {
|
||||
Add(TierConfig, bool),
|
||||
Add(Box<TierConfig>, bool),
|
||||
Edit(String, TierCreds),
|
||||
Remove(String, bool),
|
||||
Clear(bool),
|
||||
@@ -823,7 +821,7 @@ struct PrevalidatedTierCandidateMutation {
|
||||
impl TierCandidateMutation {
|
||||
fn add(mut config: TierConfig, force: bool) -> std::result::Result<Self, AdminError> {
|
||||
normalize_s3_gcs_add_tier_name(&mut config)?;
|
||||
Ok(Self::Add(config, force))
|
||||
Ok(Self::Add(Box::new(config), force))
|
||||
}
|
||||
|
||||
fn normalize_add_tier_name(&mut self) -> std::result::Result<(), AdminError> {
|
||||
@@ -910,7 +908,7 @@ impl TierCandidateMutation {
|
||||
match self {
|
||||
Self::Add(config, force) => {
|
||||
let tier_name = config.name.clone();
|
||||
candidate.add_with_deadline(config, force, deadline).await?;
|
||||
candidate.add_with_deadline(*config, force, deadline).await?;
|
||||
Ok(Some(tier_name))
|
||||
}
|
||||
Self::Edit(tier_name, credentials) => {
|
||||
@@ -2990,7 +2988,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
|
||||
let tier_type = if wasabi_version {
|
||||
TierType::Wasabi
|
||||
} else {
|
||||
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or_else(|| match ext.tier_type {
|
||||
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or(match ext.tier_type {
|
||||
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
|
||||
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
|
||||
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
|
||||
@@ -3372,28 +3370,23 @@ impl TierConfigMgr {
|
||||
|
||||
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
|
||||
self.ensure_generation_is_idle(tier_name)?;
|
||||
let d = self.get_driver(tier_name).await;
|
||||
if let Err(err) = d {
|
||||
if err.code == ERR_TIER_NOT_FOUND.code {
|
||||
return Ok(());
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
let driver = match self.get_driver(tier_name).await {
|
||||
Ok(driver) => driver,
|
||||
Err(err) if err.code == ERR_TIER_NOT_FOUND.code => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !force {
|
||||
if let Ok(driver) = d {
|
||||
match driver.in_use().await {
|
||||
Err(err) => {
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
Ok(in_use) if in_use => {
|
||||
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
|
||||
}
|
||||
_ => {}
|
||||
match driver.in_use().await {
|
||||
Err(err) => {
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
Ok(in_use) if in_use => {
|
||||
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self.tiers.remove(tier_name);
|
||||
@@ -3402,21 +3395,12 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
|
||||
let d = match self.get_driver(tier_name).await {
|
||||
Ok(d) => d,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
if let Err(err) = check_warm_backend(Some(d)).await {
|
||||
return Err(std::io::Error::other(err));
|
||||
} else {
|
||||
return Ok(());
|
||||
}
|
||||
let driver = self.get_driver(tier_name).await.map_err(std::io::Error::other)?;
|
||||
check_warm_backend(Some(driver)).await.map_err(std::io::Error::other)
|
||||
}
|
||||
|
||||
pub fn empty(&self) -> bool {
|
||||
self.list_tiers().len() == 0
|
||||
self.tiers.is_empty()
|
||||
}
|
||||
|
||||
pub fn tier_type(&self, tier_name: &str) -> String {
|
||||
@@ -3429,7 +3413,7 @@ impl TierConfigMgr {
|
||||
|
||||
pub fn list_tiers(&self) -> Vec<TierConfig> {
|
||||
let mut tier_cfgs = Vec::<TierConfig>::new();
|
||||
for (_, tier) in self.tiers.iter() {
|
||||
for tier in self.tiers.values() {
|
||||
let tier = tier.redacted();
|
||||
tier_cfgs.push(tier);
|
||||
}
|
||||
@@ -5458,7 +5442,7 @@ impl TierConfigMgr {
|
||||
let manager = handle.read().await;
|
||||
let published_digest = if intents
|
||||
.iter()
|
||||
.any(|recovered| recovered.is_peer_only_terminal() && recovered.intent.state == TierMutationIntentState::Committed)
|
||||
.any(|recovered| recovered.intent.state == TierMutationIntentState::Committed)
|
||||
{
|
||||
Some(tier_config_candidate_digest(&manager).map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
@@ -5468,18 +5452,29 @@ impl TierConfigMgr {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locally_published_committed_mutations = intents
|
||||
.iter()
|
||||
.filter(|recovered| {
|
||||
recovered.intent.state == TierMutationIntentState::Committed
|
||||
&& published_digest == Some(recovered.intent.candidate_digest)
|
||||
})
|
||||
.map(|recovered| recovered.intent.mutation_id)
|
||||
.collect::<HashSet<_>>();
|
||||
let mut prepared_mutation_blocks = HashMap::new();
|
||||
let mut committed_mutation_blocks: HashMap<String, HashSet<uuid::Uuid>> = HashMap::new();
|
||||
for recovered in intents {
|
||||
let settled_tombstone = recovered.is_peer_only_terminal()
|
||||
&& match recovered.intent.state {
|
||||
TierMutationIntentState::Aborted => true,
|
||||
TierMutationIntentState::Committed => {
|
||||
!retain_missing_mutation_blocks || published_digest == Some(recovered.intent.candidate_digest)
|
||||
}
|
||||
TierMutationIntentState::Prepared => false,
|
||||
};
|
||||
if settled_tombstone {
|
||||
// A matching in-memory manager has already crossed the local
|
||||
// publication boundary. Keep replaying and durably cleaning the
|
||||
// record, but do not re-fence object operations while that
|
||||
// terminal work finishes.
|
||||
let skip_runtime_fence = locally_published_committed_mutations.contains(&recovered.intent.mutation_id)
|
||||
|| (recovered.is_peer_only_terminal()
|
||||
&& match recovered.intent.state {
|
||||
TierMutationIntentState::Aborted => true,
|
||||
TierMutationIntentState::Committed => !retain_missing_mutation_blocks,
|
||||
TierMutationIntentState::Prepared => false,
|
||||
});
|
||||
if skip_runtime_fence {
|
||||
continue;
|
||||
}
|
||||
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, &recovered.intent)?;
|
||||
@@ -5492,6 +5487,9 @@ impl TierConfigMgr {
|
||||
}
|
||||
if retain_missing_mutation_blocks {
|
||||
for (tier_name, mutation_id) in &runtime.prepared_mutation_blocks {
|
||||
if locally_published_committed_mutations.contains(mutation_id) {
|
||||
continue;
|
||||
}
|
||||
match prepared_mutation_blocks.entry(tier_name.clone()) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(*mutation_id);
|
||||
@@ -5505,10 +5503,15 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
for (tier_name, mutation_ids) in &runtime.committed_mutation_blocks {
|
||||
committed_mutation_blocks
|
||||
.entry(tier_name.clone())
|
||||
.or_default()
|
||||
.extend(mutation_ids);
|
||||
for mutation_id in mutation_ids {
|
||||
if locally_published_committed_mutations.contains(mutation_id) {
|
||||
continue;
|
||||
}
|
||||
committed_mutation_blocks
|
||||
.entry(tier_name.clone())
|
||||
.or_default()
|
||||
.insert(*mutation_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let changed = runtime.prepared_mutation_blocks != prepared_mutation_blocks
|
||||
@@ -7116,7 +7119,8 @@ mod tests {
|
||||
let err = expect_decode_err(&encode_fixture(&wrong_hint));
|
||||
assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");
|
||||
|
||||
let poison_fields: [(&str, fn(&mut ExternalTierS3)); 6] = [
|
||||
type WasabiPoisonField = (&'static str, fn(&mut ExternalTierS3));
|
||||
let poison_fields: [WasabiPoisonField; 6] = [
|
||||
("storage_class", |s3| s3.storage_class = "GLACIER".to_string()),
|
||||
("aws_role", |s3| s3.aws_role = true),
|
||||
("web_identity_token", |s3| s3.aws_role_web_identity_token_file = "/tmp/token".to_string()),
|
||||
@@ -8237,7 +8241,11 @@ mod tests {
|
||||
peer_calls.clone(),
|
||||
Ok(PeerTierMutationState::Committed),
|
||||
)],
|
||||
TierConfigMgr::update_candidate_with_config_lock(&manager, store, TierCandidateMutation::Add(tier, true)),
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store,
|
||||
TierCandidateMutation::Add(Box::new(tier), true),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -8276,7 +8284,7 @@ mod tests {
|
||||
let add = TIER_DRIVER_TEST_FACTORY.scope(
|
||||
factory,
|
||||
apply_tier_candidate_mutation(
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-DEADLINE"), false),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-DEADLINE")), false),
|
||||
&mut candidate,
|
||||
deadline,
|
||||
),
|
||||
@@ -9095,7 +9103,9 @@ mod tests {
|
||||
fn decode_hex_fixture(hex: &str) -> Vec<u8> {
|
||||
assert_eq!(hex.len() % 2, 0, "hex fixture must contain complete bytes");
|
||||
hex.as_bytes()
|
||||
.chunks_exact(2)
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|pair| {
|
||||
let pair = std::str::from_utf8(pair).expect("hex fixture should be ASCII");
|
||||
u8::from_str_radix(pair, 16).expect("hex fixture should contain only hexadecimal digits")
|
||||
@@ -11109,7 +11119,7 @@ mod tests {
|
||||
store.clone(),
|
||||
candidate,
|
||||
version,
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
update,
|
||||
None,
|
||||
)
|
||||
@@ -11298,6 +11308,115 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn published_dual_terminal_intent_does_not_restore_local_runtime_fence_during_replay() {
|
||||
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("published tier config fixture should persist");
|
||||
let (_, current_etag) = load_tier_config_for_update(store.clone())
|
||||
.await
|
||||
.expect("published tier config fixture should load with metadata");
|
||||
let current_etag = current_etag.expect("published tier config fixture should have an ETag");
|
||||
let affected_targets = build_tier_mutation_affected_targets(
|
||||
TierMutationIntentKind::Add,
|
||||
HashSet::from(["COLD-A".to_string()]),
|
||||
&empty_mgr(),
|
||||
&persisted,
|
||||
)
|
||||
.expect("published AddTier targets should build");
|
||||
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &persisted, affected_targets)
|
||||
.expect("published AddTier intent should build")
|
||||
.expect("published AddTier should require a durable intent");
|
||||
intent
|
||||
.advance(TierMutationIntentState::Committed, Some(current_etag))
|
||||
.expect("published AddTier intent should commit");
|
||||
save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent)
|
||||
.await
|
||||
.expect("published coordinator intent should persist");
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("published peer intent should persist");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("published"));
|
||||
}
|
||||
{
|
||||
let guard = manager.read().await;
|
||||
assert_eq!(
|
||||
tier_config_candidate_digest(&guard).expect("published manager digest should build"),
|
||||
intent.candidate_digest
|
||||
);
|
||||
}
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&manager, &intent)
|
||||
.await
|
||||
.expect("pre-existing committed runtime fence should install");
|
||||
assert!(
|
||||
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await.is_err(),
|
||||
"fixture must begin with the committed runtime fence installed"
|
||||
);
|
||||
|
||||
let started = Arc::new(Notify::new());
|
||||
let release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(
|
||||
vec![Arc::new(BlockingCommitTierMutationPeer {
|
||||
started: started.clone(),
|
||||
release: release.clone(),
|
||||
})],
|
||||
async {
|
||||
let reload = TierConfigMgr::reload_handle_with(&manager, store.clone());
|
||||
tokio::pin!(reload);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::select! {
|
||||
result = &mut reload => panic!("reload finished before terminal replay was released: {result:?}"),
|
||||
_ = started.notified() => {}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("terminal replay should reach the blocking peer");
|
||||
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("terminal cleanup must not re-fence an already-published tier");
|
||||
drop(lease);
|
||||
release.add_permits(1);
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut reload)
|
||||
.await
|
||||
.expect("terminal replay should finish after the peer responds")
|
||||
.expect("terminal replay should succeed after the peer responds");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(
|
||||
TierConfigMgr::load_coordinator_mutation_intents(store.clone())
|
||||
.await
|
||||
.expect("coordinator cleanup should be readable")
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
TierConfigMgr::load_tier_mutation_intents(store)
|
||||
.await
|
||||
.expect("retained peer tombstone should be readable"),
|
||||
vec![intent]
|
||||
);
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
assert!(
|
||||
lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(),
|
||||
"retained terminal evidence must not restore the published runtime fence"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_terminal_tombstone_gc_uses_etag_and_retains_racing_replacement() {
|
||||
use crate::services::tier::tier_mutation_intent::{
|
||||
@@ -11597,8 +11716,9 @@ mod tests {
|
||||
assert!(merged[0].has_peer_record && merged[0].has_coordinator_record);
|
||||
}
|
||||
|
||||
let err = TierConfigMgr::merge_mutation_recovery_intents(&[committed.clone()], &[prepared.clone()])
|
||||
.expect_err("a peer committed record cannot outrun the coordinator commit order");
|
||||
let err =
|
||||
TierConfigMgr::merge_mutation_recovery_intents(std::slice::from_ref(&committed), std::slice::from_ref(&prepared))
|
||||
.expect_err("a peer committed record cannot outrun the coordinator commit order");
|
||||
assert!(err.to_string().contains("conflicting states"), "{err}");
|
||||
|
||||
let mut conflicting_identity = prepared.clone();
|
||||
@@ -13443,9 +13563,11 @@ mod tests {
|
||||
let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await });
|
||||
barrier.arrived.notified().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("cold driver construction must not block manager readers");
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("cold driver construction must not block manager readers"),
|
||||
);
|
||||
let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
|
||||
.await
|
||||
.expect("cold tier A construction must not block tier B")
|
||||
@@ -13648,9 +13770,11 @@ mod tests {
|
||||
let verify_manager = manager.clone();
|
||||
let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await });
|
||||
started.notified().await;
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("slow verify must not hold the manager lock");
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("slow verify must not hold the manager lock"),
|
||||
);
|
||||
release.add_permits(1);
|
||||
verify.await.expect("verify task should join").expect("verify should finish");
|
||||
}
|
||||
@@ -14058,9 +14182,11 @@ mod tests {
|
||||
vec!["COLD-A".to_string()]
|
||||
);
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(1), manager.read())
|
||||
.await
|
||||
.expect("manager reads must not wait for tier A leases");
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(1), manager.read())
|
||||
.await
|
||||
.expect("manager reads must not wait for tier A leases"),
|
||||
);
|
||||
let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
|
||||
.await
|
||||
.expect("tier B lease acquisition must not wait for tier A")
|
||||
@@ -14493,7 +14619,7 @@ mod tests {
|
||||
"https://example-compat.invalid"
|
||||
);
|
||||
let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered");
|
||||
assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none());
|
||||
assert!(!lock_unpoisoned(&runtime).generations.contains_key("COLD-A"));
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -15133,15 +15259,12 @@ mod tests {
|
||||
.filter(|object| object.bucket == bucket && object.name.starts_with(prefix))
|
||||
.cloned()
|
||||
.collect();
|
||||
objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
|
||||
objects.sort_by_key(tier_test_object_marker);
|
||||
if marker.is_some() || version_marker.is_some() {
|
||||
let marker = (marker.unwrap_or_default(), version_marker.unwrap_or_default());
|
||||
objects.retain(|object| tier_test_object_marker(object) > marker);
|
||||
}
|
||||
let limit = match usize::try_from(max_keys) {
|
||||
Ok(limit) => limit,
|
||||
Err(_) => 0,
|
||||
};
|
||||
let limit: usize = usize::try_from(max_keys).unwrap_or_default();
|
||||
let is_truncated = objects.len() > limit;
|
||||
if is_truncated {
|
||||
objects.truncate(limit);
|
||||
@@ -15171,17 +15294,16 @@ mod tests {
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<()> {
|
||||
if self.fail_reference_walk.load(Ordering::SeqCst) {
|
||||
if result
|
||||
if self.fail_reference_walk.load(Ordering::SeqCst)
|
||||
&& result
|
||||
.send(StorageObjectInfoOrErr {
|
||||
item: None,
|
||||
err: Some(Error::other("injected tier reference walk failure")),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let mut objects = self
|
||||
.listed_versions
|
||||
@@ -15192,7 +15314,7 @@ mod tests {
|
||||
.filter(|object| opts.include_free_versions || !object.transitioned_object.free_version)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
|
||||
objects.sort_by_key(tier_test_object_marker);
|
||||
if let Some(marker) = opts.marker.as_deref() {
|
||||
objects.retain(|object| object.name.as_str() > marker);
|
||||
}
|
||||
@@ -15370,17 +15492,18 @@ mod tests {
|
||||
api_view.rustfs.expect("admin RustFS payload should exist").secret_key,
|
||||
TIER_CREDENTIAL_REDACTED
|
||||
);
|
||||
let observed = lock_unpoisoned(&observed);
|
||||
assert_eq!(observed.len(), 1);
|
||||
assert_eq!(
|
||||
observed[0]
|
||||
.rustfs
|
||||
.as_ref()
|
||||
.expect("backend factory should observe the RustFS payload")
|
||||
.secret_key,
|
||||
SECRET_KEY
|
||||
);
|
||||
drop(observed);
|
||||
{
|
||||
let observed = lock_unpoisoned(&observed);
|
||||
assert_eq!(observed.len(), 1);
|
||||
assert_eq!(
|
||||
observed[0]
|
||||
.rustfs
|
||||
.as_ref()
|
||||
.expect("backend factory should observe the RustFS payload")
|
||||
.secret_key,
|
||||
SECRET_KEY
|
||||
);
|
||||
}
|
||||
|
||||
let operations = backend.op_log().await;
|
||||
assert_eq!(operations.len(), 5);
|
||||
@@ -16581,7 +16704,7 @@ mod tests {
|
||||
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
|
||||
|
||||
let targets = TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true)
|
||||
let targets = TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true)
|
||||
.affected_targets(¤t, &candidate)
|
||||
.expect("add proof should ignore unchanged durable tiers");
|
||||
assert_eq!(targets.len(), 1);
|
||||
@@ -16607,7 +16730,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store.clone(),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -16666,14 +16789,15 @@ mod tests {
|
||||
.await
|
||||
.expect("legacy nested-name Add must run the full coordinator fanout");
|
||||
|
||||
let prepared_intents = lock_unpoisoned(&prepared_intents);
|
||||
assert_eq!(prepared_intents.len(), 1);
|
||||
assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add);
|
||||
assert_eq!(prepared_intents[0].affected_targets.len(), 1);
|
||||
assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY");
|
||||
assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none());
|
||||
assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some());
|
||||
drop(prepared_intents);
|
||||
{
|
||||
let prepared_intents = lock_unpoisoned(&prepared_intents);
|
||||
assert_eq!(prepared_intents.len(), 1);
|
||||
assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add);
|
||||
assert_eq!(prepared_intents[0].affected_targets.len(), 1);
|
||||
assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY");
|
||||
assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none());
|
||||
assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some());
|
||||
}
|
||||
|
||||
let peer_calls = lock_unpoisoned(&peer_calls).clone();
|
||||
let prepare_index = peer_calls
|
||||
@@ -16755,7 +16879,7 @@ mod tests {
|
||||
let err = TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store.clone(),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true),
|
||||
)
|
||||
.await
|
||||
.expect_err("a new tier config update must wait for pending mutation recovery");
|
||||
@@ -17031,7 +17155,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -17082,7 +17206,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -17145,7 +17269,7 @@ mod tests {
|
||||
TierConfigMgr::prevalidate_candidate_owned(
|
||||
empty_mgr(),
|
||||
None,
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -17809,7 +17933,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
#[cfg(feature = "gcs")]
|
||||
@@ -719,17 +717,7 @@ async fn check_warm_backend_with_deadlines(
|
||||
if !matches!(cleanup_result, Ok(Ok(()))) {
|
||||
return Err(probe_cleanup_incomplete_error());
|
||||
}
|
||||
if let Err(err) = read_result {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
/*else if is_err_signature_does_not_match(err) {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||
}*/
|
||||
//else {
|
||||
return Err(err);
|
||||
//}
|
||||
}
|
||||
read_result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -759,7 +747,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -800,7 +788,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -820,7 +808,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -840,7 +828,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -860,7 +848,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -880,7 +868,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -900,7 +888,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -929,7 +917,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -949,7 +937,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -106,39 +104,35 @@ impl WarmBackendS3 {
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
let has_web_identity_token_file = !conf.aws_role_web_identity_token_file.is_empty();
|
||||
let has_role_arn = !conf.aws_role_arn.is_empty();
|
||||
let has_access_key = !conf.access_key.is_empty();
|
||||
let has_secret_key = !conf.secret_key.is_empty();
|
||||
|
||||
if has_web_identity_token_file != has_role_arn {
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
} else if has_access_key != has_secret_key {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
} else if conf.aws_role && (has_web_identity_token_file || has_role_arn || has_access_key || has_secret_key) {
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket == "" {
|
||||
} else if conf.bucket.is_empty() {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds: Credentials<Static>;
|
||||
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
let creds = if has_access_key && has_secret_key {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
creds = Credentials::new(Static(Value {
|
||||
Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
}))
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
}
|
||||
};
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let opts = Options {
|
||||
creds,
|
||||
@@ -162,11 +156,11 @@ impl WarmBackendS3 {
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix, object)
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_with_result(&self, object: &str, rv: &str) -> Result<RemoveObjectResult, std::io::Error> {
|
||||
@@ -413,6 +407,10 @@ impl TransitionCandidateVersions {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
clippy::items_after_test_module,
|
||||
reason = "keep parsing tests adjacent to the helpers they cover"
|
||||
)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
@@ -917,7 +915,7 @@ impl WarmBackend for WarmBackendS3 {
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
Ok(!result.common_prefixes.is_empty() || !result.contents.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -883,6 +883,17 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionKillPoint as SetDiskTransitionTransactionKillPoint,
|
||||
TransitionTransactionKillPointBarrier as SetDiskTransitionTransactionKillPointBarrier,
|
||||
};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionMutationKind as SetDiskTransitionTransactionMutationKind,
|
||||
TransitionTransactionMutationObservation as SetDiskTransitionTransactionMutationObservation,
|
||||
TransitionTransactionMutationProbe as SetDiskTransitionTransactionMutationProbe,
|
||||
};
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
|
||||
@@ -3301,13 +3301,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
||||
Self::assign_rename_data_indexes(&mut parts_metadatas);
|
||||
let mut rename_result = SetDisks::rename_data_owned(
|
||||
// Disk deadlines can expire before physical publication or failure undo drains.
|
||||
let mut rename_result = SetDisks::rename_data_owned_with_fence(
|
||||
&commit_disks,
|
||||
(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path),
|
||||
parts_metadatas,
|
||||
(&commit_bucket, &commit_object),
|
||||
write_quorum,
|
||||
commit_allows_early_ack,
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(write_quorum, None)
|
||||
.with_namespace_commit_guard(
|
||||
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
if let Ok(rename_commit) = rename_result.as_mut() {
|
||||
@@ -6758,6 +6763,301 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_advances_namespace_generation_after_commit() {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-namespace-commit";
|
||||
let object = "completed-object";
|
||||
let body = vec![0x65; 4096];
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let before = set_disks.ctx.namespace_commit_generation();
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before, "staging is not publication");
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("completion must finish")
|
||||
.expect("the four real shards must commit");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET after completion must finish")
|
||||
.expect("a successful completion must be immediately readable");
|
||||
let mut actual = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("the completed body stream must finish")
|
||||
.expect("read completed body");
|
||||
assert_eq!(actual, body);
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists());
|
||||
}
|
||||
assert!(matches!(
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
assert_eq!(
|
||||
set_disks.ctx.namespace_commit_generation(),
|
||||
before + 2,
|
||||
"one completed MPU must invalidate snapshots at namespace admission and physical retirement"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_complete_multipart_physical_namespace_owner(undo: bool) {
|
||||
use crate::disk::os::{self, prepared_publication_test_hooks as hooks};
|
||||
use crate::set_disk::core::io_primitives::rename_fault_injection;
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-physical-namespace";
|
||||
let object = if undo { "undo-tail" } else { "publication-tail" };
|
||||
let old_body = vec![0x41; 1024];
|
||||
let new_body = vec![0x62; 4096];
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let old = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(old_body.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed a real readable old version");
|
||||
let old_etag = old.etag.expect("the committed old object must have an ETag");
|
||||
let before = set_disks.ctx.namespace_commit_generation();
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &new_body, &ObjectOptions::default()).await;
|
||||
let new_etag = get_complete_multipart_md5(&parts);
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for dir in &dirs {
|
||||
assert!(dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists());
|
||||
}
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before);
|
||||
let _fault = undo.then(|| rename_fault_injection::fail_rename_on(object, &[2, 3]));
|
||||
let stage = if undo {
|
||||
hooks::Stage::Rename
|
||||
} else {
|
||||
hooks::Stage::PreparedRename
|
||||
};
|
||||
let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let mut hooks = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut mutation_paths = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical MPU fixture requires local disks");
|
||||
};
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, object)
|
||||
.expect("leased IO path");
|
||||
let entered_tx = entered_tx.clone();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(hooks::install_at(stage, &destination.join(STORAGE_FORMAT_FILE), move || {
|
||||
let _ = entered_tx.send(index);
|
||||
// Dropping senders releases every syscall on assertion failure, too.
|
||||
let _ = release_rx.recv();
|
||||
}));
|
||||
releases.push(release_tx);
|
||||
// Canonical rename serializes the object directory; backup restore
|
||||
// serializes its xl.meta destination. Drain the actual executor key.
|
||||
mutation_paths.push(if undo {
|
||||
destination.join(STORAGE_FORMAT_FILE)
|
||||
} else {
|
||||
destination
|
||||
});
|
||||
}
|
||||
drop(entered_tx);
|
||||
let complete_set = set_disks.clone();
|
||||
let complete_upload = upload_id.clone();
|
||||
let mut complete = tokio::spawn(async move {
|
||||
complete_set
|
||||
.complete_multipart_upload(bucket, object, &complete_upload, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
let mut complete_joined = false;
|
||||
let mut observed_counts = None;
|
||||
let observations = std::panic::AssertUnwindSafe(async {
|
||||
let expected_publishers = if undo { 2 } else { 4 };
|
||||
let entered = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let mut entered = HashSet::new();
|
||||
while entered.len() < expected_publishers {
|
||||
tokio::select! {
|
||||
index = entered_rx.recv() => {
|
||||
assert!(entered.insert(index.expect("physical publisher must signal entry")));
|
||||
}
|
||||
result = &mut complete => {
|
||||
complete_joined = true;
|
||||
panic!("completion returned before physical entry: {result:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
entered
|
||||
})
|
||||
.await
|
||||
.expect("all expected physical metadata operations must enter");
|
||||
let pending_at_entry = set_disks.ctx.namespace_commits_pending();
|
||||
let generation_at_entry = set_disks.ctx.namespace_commit_generation();
|
||||
for &index in &entered {
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
disks[index].read_version("", bucket, object, "", &ReadOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("metadata observation must finish while publication is paused")
|
||||
.expect("metadata before the paused physical action must be readable");
|
||||
assert_eq!(
|
||||
metadata.metadata.get("etag"),
|
||||
Some(if undo { &new_etag } else { &old_etag }),
|
||||
"undo must follow actual publication; prepared rename must precede publication"
|
||||
);
|
||||
}
|
||||
|
||||
// The entry signals run inside the real blocking closures, after each
|
||||
// wrapper installed its normal deadline. No quota/external guard disables it.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let joined = tokio::time::timeout(Duration::from_secs(5), &mut complete).await;
|
||||
complete_joined = joined.is_ok();
|
||||
let result = joined
|
||||
.expect("ordinary MPU disk/undo deadlines must still return before physical drain")
|
||||
.expect("completion task must not panic");
|
||||
assert!(result.is_err(), "a timed-out or two-shard commit cannot acknowledge success");
|
||||
let pending_after_timeout = set_disks.ctx.namespace_commits_pending();
|
||||
let generation_after_timeout = set_disks.ctx.namespace_commit_generation();
|
||||
observed_counts = Some((pending_at_entry, generation_at_entry, pending_after_timeout, generation_after_timeout));
|
||||
for &index in &entered {
|
||||
assert!(
|
||||
os::acquire_rename_data_mutation_lease(&disks[index].path(), bucket, &mutation_paths[index])
|
||||
.now_or_never()
|
||||
.is_none(),
|
||||
"timed-out physical work must still own its object serialization"
|
||||
);
|
||||
}
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists(),
|
||||
"failed completion must not clean the upload staging"
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
|
||||
// Release even after a failed observation, then finish dispatch before
|
||||
// draining every physical key. No per-disk assertion may skip a later drain.
|
||||
drop(releases);
|
||||
drop(hooks);
|
||||
let coordinator_drained = complete_joined
|
||||
|| tokio::time::timeout(Duration::from_secs(10), &mut complete).await.is_ok();
|
||||
if !coordinator_drained {
|
||||
complete.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), &mut complete).await;
|
||||
}
|
||||
let drains = futures::future::join_all(disks.iter().zip(&mutation_paths).map(|(disk, destination)| async move {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, destination),
|
||||
)
|
||||
.await
|
||||
.map(drop)
|
||||
}))
|
||||
.await;
|
||||
let owner_drained = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while set_disks.ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let physical_drained = drains.iter().all(|drain| drain.is_ok());
|
||||
if !coordinator_drained || !physical_drained || owner_drained.is_err() {
|
||||
// A bounded cleanup failure cannot justify deleting roots that a
|
||||
// detached executor might still use. Keep them for diagnosis.
|
||||
let retained = dirs.into_iter().map(TempDir::keep).collect::<Vec<_>>();
|
||||
eprintln!("MPU cleanup incomplete: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
panic!("MPU cleanup did not drain: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
}
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// Preserve the original drain checks after collecting every result.
|
||||
for drained in drains {
|
||||
drained.expect("released physical MPU work must drain");
|
||||
}
|
||||
owner_drained.expect("physical retirement must finish its namespace counter decrement");
|
||||
for disk in &disks {
|
||||
let metadata = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("all disks must expose the expected final metadata");
|
||||
assert_eq!(metadata.metadata.get("etag"), Some(if undo { &old_etag } else { &new_etag }));
|
||||
}
|
||||
let (pending_at_entry, generation_at_entry, pending_after_timeout, generation_after_timeout) =
|
||||
observed_counts.expect("successful observations must record namespace counters");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET after physical drain must finish")
|
||||
.expect("the real final object must be readable");
|
||||
let mut actual = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("the final object stream must finish")
|
||||
.expect("read final object bytes");
|
||||
assert_eq!(actual, if undo { old_body } else { new_body });
|
||||
assert!(
|
||||
pending_at_entry && pending_after_timeout,
|
||||
"physical MPU work outlived namespace accounting: undo={undo}"
|
||||
);
|
||||
assert_eq!(generation_at_entry, before + 1);
|
||||
assert_eq!(
|
||||
generation_after_timeout, generation_at_entry,
|
||||
"the blocked physical owner cannot retire early"
|
||||
);
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before + 2);
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_complete_multipart_physical_namespace_owner(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_failed_quorum_keeps_namespace_owner_until_physical_undo() {
|
||||
assert_complete_multipart_physical_namespace_owner(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -630,27 +630,14 @@ impl ECStore {
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
let (meta, pool_meta_replica_state) = {
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
load.await?
|
||||
load_pool_meta_for_startup(self.pools.clone(), &mut write_state).await?
|
||||
};
|
||||
let update = meta.validate(self.pools.clone())?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": pool_meta_replica_state.needs_repair,
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
@@ -662,17 +649,15 @@ impl ECStore {
|
||||
// distributed startup can race on the same lock and replay the prior init bug.
|
||||
{
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
let persist = persist_pool_meta_for_startup_if_safe(
|
||||
installed_pool_meta = persist_pool_meta_for_startup_if_safe(
|
||||
&installed_pool_meta,
|
||||
self.pools.clone(),
|
||||
pool_meta_replica_state,
|
||||
&mut write_state,
|
||||
update,
|
||||
should_persist_pool_meta,
|
||||
);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
|
||||
installed_pool_meta = persist.await?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
{
|
||||
@@ -843,7 +828,17 @@ mod tests {
|
||||
recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
|
||||
IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
|
||||
observe_recovery_source, save_recovery_control_if_absent,
|
||||
observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
|
||||
},
|
||||
recovery_disposition::{
|
||||
IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
|
||||
dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
|
||||
load_recovery_disposition,
|
||||
},
|
||||
recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
|
||||
recovery_export::{
|
||||
create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
|
||||
recovery_export_record_object_name,
|
||||
},
|
||||
tier_delete_journal::{
|
||||
DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
|
||||
@@ -867,9 +862,10 @@ mod tests {
|
||||
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
load_transition_transaction_record, recover_transition_transaction_records,
|
||||
recover_transition_transaction_records_at, save_transition_transaction_record,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, recover_transition_transaction_records_at,
|
||||
retry_transition_recovery_for_operator, save_transition_transaction_record,
|
||||
save_transition_transaction_record_if_current, transition_recovery_control_id,
|
||||
transition_transaction_record_object_name,
|
||||
},
|
||||
@@ -881,7 +877,11 @@ mod tests {
|
||||
data_movement::SourceCleanupDeleteBarrier,
|
||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
|
||||
runtime::{global::set_object_store_resolver, sources as runtime_sources},
|
||||
services::notification_sys::acquire_tier_delete_journal_fleet_proof,
|
||||
services::notification_sys::{
|
||||
acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
|
||||
install_transition_transaction_compaction_fleet_proof_for_test,
|
||||
transition_transaction_compaction_fleet_proof_matches,
|
||||
},
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{
|
||||
@@ -903,7 +903,14 @@ mod tests {
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend},
|
||||
},
|
||||
set_disk::SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
set_disk::{
|
||||
SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
|
||||
SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
|
||||
SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
|
||||
SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
|
||||
SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
|
||||
SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
},
|
||||
storage_api_contracts::list::ListOperations as _,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -4384,7 +4391,7 @@ mod tests {
|
||||
}
|
||||
retry_source_info.parts = Arc::new(retry_source_parts);
|
||||
assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
|
||||
assert!(!retry_source_info.is_multipart());
|
||||
assert!(retry_source_info.is_multipart());
|
||||
assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
|
||||
assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
|
||||
assert!(
|
||||
@@ -11590,6 +11597,36 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
|
||||
let records = store
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX,
|
||||
None,
|
||||
None,
|
||||
100,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("transition transaction records should be listable")
|
||||
.objects;
|
||||
assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
|
||||
let transaction_id = records[0]
|
||||
.name
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.and_then(|name| uuid::Uuid::parse_str(name).ok())
|
||||
.expect("transition transaction path should end in its UUID");
|
||||
load_transition_transaction_record(store, transaction_id)
|
||||
.await
|
||||
.expect("transition transaction should load")
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_transition_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
@@ -16885,6 +16922,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
|
||||
let mut created_exports = Vec::new();
|
||||
for exportable in first_controls
|
||||
.iter()
|
||||
.filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
|
||||
{
|
||||
let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
|
||||
.await
|
||||
.expect("fresh legacy recovery observation should be exportable");
|
||||
let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
|
||||
.await
|
||||
.expect("legacy recovery export should be created exactly once");
|
||||
assert!(!created.replayed);
|
||||
let loaded = load_recovery_export(store.clone(), &created.export_id)
|
||||
.await
|
||||
.expect("created legacy recovery export should load");
|
||||
assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
|
||||
let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
|
||||
.await
|
||||
.expect("the same observed generation should replay its immutable export");
|
||||
assert!(replayed.replayed);
|
||||
assert_eq!(replayed.encoded, created.encoded);
|
||||
created_exports.push(created);
|
||||
}
|
||||
assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
|
||||
|
||||
let corrupt_export_id = &created_exports[0].export_id;
|
||||
let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
|
||||
.expect("export path should build");
|
||||
com::save_config(store.clone(), &export_path, Vec::new())
|
||||
.await
|
||||
.expect("zero-byte corruption fixture should persist");
|
||||
let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
|
||||
.await
|
||||
.expect_err("an existing zero-byte export must fail closed");
|
||||
assert!(!matches!(corrupt_export, Error::ConfigNotFound));
|
||||
|
||||
com::save_config(
|
||||
store.clone(),
|
||||
&journal_paths[0],
|
||||
@@ -16910,6 +16984,301 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "LEGACY-DISPOSITION";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("legacy disposition tier lease should resolve")
|
||||
.backend_identity();
|
||||
let fixtures = [
|
||||
serde_json::json!({
|
||||
"version": 1,
|
||||
"obj_name": "legacy/disposition-v1",
|
||||
"version_id": "opaque-disposition-v1",
|
||||
"tier_name": tier_name,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": 2,
|
||||
"obj_name": "legacy/disposition-v2",
|
||||
"version_id": "opaque-disposition-v2",
|
||||
"tier_name": tier_name,
|
||||
"backend_identity": backend_identity,
|
||||
}),
|
||||
];
|
||||
let mut journal_paths = Vec::new();
|
||||
for fixture in &fixtures {
|
||||
let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
|
||||
let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
|
||||
.expect("legacy disposition fixture should decode");
|
||||
let path = tier_delete_journal_object_name(&entry);
|
||||
com::save_config(store.clone(), &path, data)
|
||||
.await
|
||||
.expect("legacy disposition fixture should persist");
|
||||
journal_paths.push(path);
|
||||
}
|
||||
|
||||
let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("legacy disposition recovery scan should finish");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
|
||||
|
||||
let mut controls = list_recovery_controls(
|
||||
store.clone(),
|
||||
IlmRecoveryProtocol::TierDeleteJournal,
|
||||
Some(IlmRecoveryClassification::RetainedAmbiguous),
|
||||
100,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("legacy disposition controls should be listable")
|
||||
.records;
|
||||
controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
|
||||
assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
|
||||
|
||||
let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
|
||||
let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
|
||||
let wrong_export_sha256 = "ff".repeat(32);
|
||||
for (index, control) in controls.iter().enumerate() {
|
||||
let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
|
||||
.await
|
||||
.expect("legacy disposition source should be observable");
|
||||
let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
|
||||
.await
|
||||
.expect("legacy disposition export should persist");
|
||||
let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.expect("legacy disposition timestamp should fit i64");
|
||||
|
||||
if index == 0 {
|
||||
let wrong_hash = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&wrong_export_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("a mismatched export checksum must fail before local deletion");
|
||||
assert_eq!(wrong_hash, Error::PreconditionFailed);
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
|
||||
}
|
||||
|
||||
let dry_run = dry_run_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
)
|
||||
.await
|
||||
.expect("legacy disposition dry-run should validate exact local state");
|
||||
assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
|
||||
assert_eq!(
|
||||
tier_delete_journal_count(store.clone()).await,
|
||||
fixtures.len() - index,
|
||||
"dry-run must not delete a legacy journal"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
|
||||
.await,
|
||||
Err(Error::ConfigNotFound)
|
||||
),
|
||||
"dry-run must not persist a disposition record"
|
||||
);
|
||||
|
||||
if index == 0 {
|
||||
inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
|
||||
Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("the injected crash must stop after local delete commits");
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
|
||||
let interrupted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
|
||||
.await
|
||||
.expect("the applying disposition must survive the post-delete crash");
|
||||
assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
|
||||
assert!(
|
||||
interrupted.disposition.confirmed_absent.is_empty(),
|
||||
"the crash must occur before absence progress is persisted"
|
||||
);
|
||||
} else {
|
||||
inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
|
||||
Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("the injected crash must stop after control abandonment commits");
|
||||
let interrupted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
|
||||
.await
|
||||
.expect("the applying disposition must survive the post-control crash");
|
||||
assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
|
||||
assert_eq!(
|
||||
interrupted.disposition.confirmed_absent.len(),
|
||||
interrupted.disposition.identity.source_generation.copies.len()
|
||||
);
|
||||
|
||||
let abandoned =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.await
|
||||
.expect("the abandoned control must survive the injected crash");
|
||||
let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
|
||||
let mut wrong_history = abandoned.control;
|
||||
wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
|
||||
let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
|
||||
.expect("control path should remain canonical");
|
||||
com::save_config(
|
||||
store.clone(),
|
||||
&control_path,
|
||||
wrong_history
|
||||
.encode()
|
||||
.expect("the alternate valid control history should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("the alternate control history fixture should persist");
|
||||
let wrong_history_err = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos + 1,
|
||||
))
|
||||
.await
|
||||
.expect_err("a different abandoned control history must not bridge to completion");
|
||||
assert_eq!(wrong_history_err, Error::PreconditionFailed);
|
||||
com::save_config(store.clone(), &control_path, exact_abandoned)
|
||||
.await
|
||||
.expect("the exact abandoned control fixture should be restored");
|
||||
}
|
||||
|
||||
let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
|
||||
let executed = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
replay_confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect("a later request must resume and complete the interrupted disposition");
|
||||
assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
|
||||
assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
|
||||
assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
|
||||
assert!(matches!(
|
||||
com::read_config(store.clone(), &observation.canonical_source_path).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
if index == 0 {
|
||||
let untouched = journal_paths
|
||||
.iter()
|
||||
.find(|path| *path != &observation.canonical_source_path)
|
||||
.expect("the other legacy journal should remain");
|
||||
com::read_config(store.clone(), untouched)
|
||||
.await
|
||||
.expect("disposition must not remove a different legacy journal");
|
||||
}
|
||||
|
||||
let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.await
|
||||
.expect("abandoned recovery control should remain inspectable");
|
||||
assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
|
||||
assert_eq!(abandoned.control.revision, observation.control_revision + 1);
|
||||
assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
|
||||
|
||||
let persisted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
|
||||
.await
|
||||
.expect("completed disposition should remain durable");
|
||||
assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
|
||||
|
||||
let replayed = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
replay_confirmed_at_unix_nanos + 1,
|
||||
))
|
||||
.await
|
||||
.expect("same actor should replay the completed disposition");
|
||||
assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
|
||||
assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
|
||||
|
||||
let wrong_actor = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&wrong_actor_sha256,
|
||||
replay_confirmed_at_unix_nanos + 2,
|
||||
))
|
||||
.await
|
||||
.expect_err("a different actor must not replay a completed disposition");
|
||||
assert_eq!(wrong_actor, Error::PreconditionFailed);
|
||||
|
||||
assert!(
|
||||
!Box::pin(garbage_collect_completed_recovery_disposition(
|
||||
store.clone(),
|
||||
&persisted,
|
||||
persisted.disposition.retain_until_unix_nanos - 1,
|
||||
))
|
||||
.await
|
||||
.expect("completed disposition should remain before retention expires")
|
||||
);
|
||||
assert!(
|
||||
Box::pin(garbage_collect_completed_recovery_disposition(
|
||||
store.clone(),
|
||||
&persisted,
|
||||
persisted.disposition.retain_until_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect("expired completed disposition should be garbage collected")
|
||||
);
|
||||
assert!(matches!(
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
|
||||
assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
|
||||
assert!(
|
||||
backend.op_log().await.is_empty(),
|
||||
"legacy disposition must not invoke any backend operation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -19257,6 +19626,330 @@ mod tests {
|
||||
assert_eq!(loaded_ids, intent_ids);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
|
||||
(
|
||||
observations.len(),
|
||||
observations.iter().map(|observation| observation.encoded_bytes).sum(),
|
||||
observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
|
||||
let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "MUTATION-MEASURE";
|
||||
register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-mutation-measurement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("measurement bucket should be created");
|
||||
|
||||
let run_case = |profile: &'static str, size: usize| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let object = format!("{profile}-{size}.bin");
|
||||
let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
|
||||
let original = store
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("measurement source should be written");
|
||||
let probe = TransitionTransactionMutationProbe::install(bucket, &object);
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: original.etag.clone().expect("measurement source should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("measurement transition should commit");
|
||||
probe.observations()
|
||||
}
|
||||
};
|
||||
|
||||
let sizes = [4 * 1024, 1024 * 1024];
|
||||
let mut legacy = Vec::with_capacity(sizes.len());
|
||||
for size in sizes {
|
||||
legacy.push((size, run_case("legacy", size).await));
|
||||
}
|
||||
|
||||
let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
|
||||
assert_eq!(size, compact_size);
|
||||
let compact_observations = run_case("compact", size).await;
|
||||
let legacy_measurement = transition_mutation_measurement(&legacy_observations);
|
||||
let compact_measurement = transition_mutation_measurement(&compact_observations);
|
||||
assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
|
||||
assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
|
||||
assert!(
|
||||
compact_measurement.1 < legacy_measurement.1,
|
||||
"compact transaction bodies should write fewer aggregate bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
legacy_observations
|
||||
.iter()
|
||||
.map(|observation| (observation.kind, observation.previous_state, observation.state))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadStarted),
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadOutcomeUnknown),
|
||||
TransitionTransactionState::Uploaded,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::Uploaded),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::LocalCommitStarted),
|
||||
TransitionTransactionState::Committed,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::Delete,
|
||||
Some(TransitionTransactionState::Committed),
|
||||
TransitionTransactionState::Committed,
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
compact_observations
|
||||
.iter()
|
||||
.map(|observation| (observation.kind, observation.previous_state, observation.state))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(
|
||||
TransitionTransactionMutationKind::Create,
|
||||
None,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadOutcomeUnknown),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::Delete,
|
||||
Some(TransitionTransactionState::LocalCommitStarted),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
]
|
||||
);
|
||||
assert!(legacy_observations.iter().all(|observation| observation.succeeded));
|
||||
assert!(compact_observations.iter().all(|observation| observation.succeeded));
|
||||
println!(
|
||||
"transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
|
||||
legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
|
||||
);
|
||||
println!(
|
||||
"transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
|
||||
compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
|
||||
);
|
||||
}
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
let admitted = acquire_transition_transaction_compaction_fleet_proof()
|
||||
.expect("published homogeneous proof should admit one compact writer");
|
||||
assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
|
||||
drop(compaction_proof);
|
||||
assert!(
|
||||
!transition_transaction_compaction_fleet_proof_matches(&admitted),
|
||||
"revocation must fence a writer admitted by the previous process-epoch snapshot"
|
||||
);
|
||||
drop(admitted);
|
||||
assert!(
|
||||
acquire_transition_transaction_compaction_fleet_proof().is_none(),
|
||||
"revoking the homogeneous proof must restore the legacy writer profile"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
|
||||
let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "COMPACT-KILL";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("mock tier lease should remain available during recovery");
|
||||
let bucket = "compact-transition-kill-points";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("kill-point bucket should be created");
|
||||
let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
|
||||
for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
|
||||
(
|
||||
0,
|
||||
TransitionTransactionKillPoint::PrePutFence,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
(
|
||||
1,
|
||||
TransitionTransactionKillPoint::UploadBeforeCommitFence,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
(
|
||||
2,
|
||||
TransitionTransactionKillPoint::LocalCommitBeforeDelete,
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
2,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
(
|
||||
3,
|
||||
TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
2,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let object = format!("kill-point-{index}.bin");
|
||||
let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let source = store
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("kill-point source should be written");
|
||||
let remote_before = backend.object_count().await;
|
||||
let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
|
||||
let transition_store = store.clone();
|
||||
let transition_object = object.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(
|
||||
bucket,
|
||||
&transition_object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("kill-point source should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: source.version_id.map(|version| version.to_string()),
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
transition.abort();
|
||||
assert!(
|
||||
transition
|
||||
.await
|
||||
.expect_err("kill-point transition should be cancelled")
|
||||
.is_cancelled(),
|
||||
"kill-point transition should stop without unwinding"
|
||||
);
|
||||
drop(barrier);
|
||||
|
||||
let transaction = only_transition_transaction(store.clone()).await;
|
||||
assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
|
||||
let paused_source = store
|
||||
.get_object_info(
|
||||
bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("kill-point source metadata should remain readable");
|
||||
assert_eq!(
|
||||
paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
|
||||
expected_committed
|
||||
);
|
||||
let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
|
||||
assert_eq!(backend.object_count().await, expected_remote_at_pause);
|
||||
|
||||
let stats = recover_transition_transaction_records_at(
|
||||
store.clone(),
|
||||
100,
|
||||
None,
|
||||
i128::from(transaction.not_after_unix_nanos) + 1,
|
||||
)
|
||||
.await
|
||||
.expect("kill-point transaction recovery should complete");
|
||||
if expected_recovered {
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
} else {
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
|
||||
assert_eq!(
|
||||
transition_transaction_record_count(store.clone()).await,
|
||||
1,
|
||||
"an uncommitted local-commit fence must retain its exact remote owner"
|
||||
);
|
||||
}
|
||||
let expected_remote_after_recovery = if matches!(
|
||||
point,
|
||||
TransitionTransactionKillPoint::LocalCommitBeforeDelete
|
||||
| TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
|
||||
) {
|
||||
remote_before + 1
|
||||
} else {
|
||||
remote_before
|
||||
};
|
||||
assert_eq!(backend.object_count().await, expected_remote_after_recovery);
|
||||
assert_eq!(backend.remove_count().await, usize::from(index >= 1));
|
||||
|
||||
let mut restored = Vec::new();
|
||||
store
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("kill-point source should remain readable through its authoritative location")
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("kill-point source body should drain");
|
||||
assert_eq!(restored, payload);
|
||||
|
||||
if !expected_recovered {
|
||||
break;
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -20302,7 +20995,7 @@ mod tests {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
data_dir: original.data_dir.expect("source object should have data_dir"),
|
||||
mod_time_unix_nanos: original
|
||||
.mod_time
|
||||
.expect("source object should have mod_time")
|
||||
@@ -20436,7 +21129,7 @@ mod tests {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
data_dir: original.data_dir.expect("source object should have data_dir"),
|
||||
mod_time_unix_nanos: original
|
||||
.mod_time
|
||||
.expect("source object should have mod_time")
|
||||
@@ -20577,10 +21270,77 @@ mod tests {
|
||||
IlmRecoveryClassification::RetainedAmbiguous
|
||||
);
|
||||
let local_commit_control =
|
||||
load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
.await
|
||||
.expect("local-commit control should persist");
|
||||
assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
|
||||
let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
|
||||
.await
|
||||
.expect("retained upload should be inspectable for a bounded retry");
|
||||
let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
|
||||
.await
|
||||
.expect("operator-required local commit should be inspectable for a bounded retry");
|
||||
assert!(upload_status.retry_ready);
|
||||
assert!(local_status.retry_ready);
|
||||
assert!(matches!(
|
||||
retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&upload_started_control_id,
|
||||
upload_status.control_revision + 1,
|
||||
&upload_status.source_generation_sha256,
|
||||
)
|
||||
.await,
|
||||
Err(TransitionOperatorError::StaleRecoveryControl)
|
||||
));
|
||||
|
||||
let put_count_before_retry = backend.put_count().await;
|
||||
let get_count_before_retry = backend.get_count().await;
|
||||
let remove_count_before_retry = backend.remove_count().await;
|
||||
let upload_retry = retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&upload_started_control_id,
|
||||
upload_status.control_revision,
|
||||
&upload_status.source_generation_sha256,
|
||||
)
|
||||
.await
|
||||
.expect("exact retained upload generation should be rearmed");
|
||||
let local_retry = retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&local_commit_control_id,
|
||||
local_status.control_revision,
|
||||
&local_status.source_generation_sha256,
|
||||
)
|
||||
.await
|
||||
.expect("exact operator-required local commit generation should be rearmed");
|
||||
assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
|
||||
assert_eq!(local_retry.attempt_count, local_status.attempt_count);
|
||||
assert_eq!(backend.put_count().await, put_count_before_retry);
|
||||
assert_eq!(backend.get_count().await, get_count_before_retry);
|
||||
assert_eq!(backend.remove_count().await, remove_count_before_retry);
|
||||
assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
|
||||
|
||||
let retried = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("rearmed records should be re-evaluated through normal recovery");
|
||||
assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
|
||||
let upload_retained =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
|
||||
.await
|
||||
.expect("upload retry result should persist");
|
||||
let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
.await
|
||||
.expect("local commit retry result should persist");
|
||||
assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
|
||||
assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
|
||||
assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
|
||||
assert_eq!(backend.put_count().await, put_count_before_retry);
|
||||
assert_eq!(backend.get_count().await, get_count_before_retry);
|
||||
assert_eq!(backend.remove_count().await, remove_count_before_retry);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -20841,6 +21601,7 @@ mod tests {
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
@@ -20909,6 +21670,7 @@ mod tests {
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
"a response-lost PUT must not remain in UploadStarted"
|
||||
);
|
||||
assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
|
||||
assert!(
|
||||
backend.contains(&transaction.remote_object).await,
|
||||
"the test backend must retain the remote candidate"
|
||||
|
||||
@@ -442,7 +442,7 @@ pub(crate) mod utils;
|
||||
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
@@ -1787,7 +1787,7 @@ mod tests {
|
||||
|
||||
// Build a minimal ECStore carrying an explicit instance context. Empty
|
||||
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
|
||||
pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
let endpoint_pools = EndpointServerPools::default();
|
||||
Arc::new(ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
|
||||
@@ -13,10 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::utils::has_bad_path_component;
|
||||
use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use tracing::{debug, error};
|
||||
|
||||
@@ -25,203 +22,6 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
|
||||
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
|
||||
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
|
||||
|
||||
/// An instance-bound capability for internal writes before ECStore/IAM startup.
|
||||
/// Its private context and volume checks cannot be replaced by a caller guard.
|
||||
#[derive(Clone)]
|
||||
pub struct BootstrapLocalTarget {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
impl BootstrapLocalTarget {
|
||||
pub fn new(ctx: Arc<InstanceContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub fn is_for_store(&self, store: &ECStore) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, &store.ctx)
|
||||
}
|
||||
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if scanner_token.is_some() {
|
||||
return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
|
||||
}
|
||||
validate_bootstrap_volume(source.0)?;
|
||||
validate_bootstrap_volume(destination.0)?;
|
||||
rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
validate_bootstrap_volume(volume)?;
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
// Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
|
||||
// Validate both raw rename volumes before any disk lookup or admission.
|
||||
if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Execute on this instance's active local disk through the physical owner.
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
|
||||
Some(Arc::new(
|
||||
self.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.map_err(|err| DiskError::other(err.to_string()))?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
// The optional ID is a cold lookup to cache only after final admission.
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
|
||||
if let Some(path) = cached_path {
|
||||
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
&& disk.get_disk_id().await? == Some(disk_id)
|
||||
{
|
||||
return Ok((disk, None));
|
||||
}
|
||||
}
|
||||
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
|
||||
// Disk identity may perform format I/O. No registry guard spans this await.
|
||||
for disk in disks {
|
||||
if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
|
||||
return Ok((disk, Some(disk_id)));
|
||||
}
|
||||
}
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
volume: &str,
|
||||
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
|
||||
if !matches!(disk.as_ref(), Disk::Local(_)) {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
let map = ctx.local_disk_map();
|
||||
let active = map.read().await;
|
||||
if !active
|
||||
.get(&disk.endpoint().to_string())
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, disk))
|
||||
{
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
// Preserve registry -> ID-cache lock order; no filesystem I/O under either.
|
||||
if let Some(disk_id) = disk_id {
|
||||
ctx.local_disk_id_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk_id, disk.endpoint().to_string());
|
||||
}
|
||||
// Admission linearizes under the registry read: replacement/quarantine
|
||||
// before this point rejects; later changes do not revoke physical I/O.
|
||||
Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit()))
|
||||
}
|
||||
|
||||
async fn rename_local_data_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?;
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn undo_local_write_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
if !opts.undo_write {
|
||||
return Err(DiskError::other("target undo requires undo_write"));
|
||||
}
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?;
|
||||
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
|
||||
.await;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
|
||||
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
|
||||
}
|
||||
@@ -465,522 +265,6 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
|
||||
let mut format = crate::layout::format::FormatV3::new(1, 1);
|
||||
format.erasure.this = id;
|
||||
format.erasure.sets[0][0] = id;
|
||||
let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("create format volume");
|
||||
tokio::fs::write(
|
||||
meta.join(crate::disk::FORMAT_CONFIG_FILE),
|
||||
serde_json::to_vec(&format).expect("encode format"),
|
||||
)
|
||||
.await
|
||||
.expect("write real disk identity");
|
||||
let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open real local disk");
|
||||
assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
disk
|
||||
}
|
||||
|
||||
fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 1, 0);
|
||||
fi.erasure.index = 1;
|
||||
fi.version_id = Some(version);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.size = i64::try_from(body.len()).expect("fixture length");
|
||||
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||
number: 1,
|
||||
size: body.len(),
|
||||
actual_size: fi.size,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.data = Some(bytes::Bytes::from_static(body));
|
||||
fi.set_inline_data();
|
||||
fi
|
||||
}
|
||||
|
||||
async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
|
||||
let dir = disk.path().join(volume);
|
||||
tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
|
||||
disk.write_metadata(volume, volume, object, fi.clone())
|
||||
.await
|
||||
.expect("seed real metadata");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
object,
|
||||
&fi.version_id.expect("fixture version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read fixture before mutation");
|
||||
assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
|
||||
tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
let ctx_a = Arc::new(InstanceContext::new());
|
||||
let ctx_b = Arc::new(InstanceContext::new());
|
||||
let a = tempfile::tempdir().expect("A root");
|
||||
let b = tempfile::tempdir().expect("B root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk_a = target_disk(&ctx_a, a.path(), id).await;
|
||||
let disk_b = target_disk(&ctx_b, b.path(), id).await;
|
||||
if warm {
|
||||
assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
|
||||
assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
|
||||
}
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("destination", version, b"new-A");
|
||||
for disk in [&disk_a, &disk_b] {
|
||||
seed_target(disk, "target-bucket", "staged", fi.clone()).await;
|
||||
}
|
||||
let b_before = seed_target(
|
||||
&disk_b,
|
||||
"target-bucket",
|
||||
"destination",
|
||||
target_file_info("destination", version, b"old-B"),
|
||||
)
|
||||
.await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
|
||||
store
|
||||
.rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
|
||||
.await
|
||||
.expect("rename on A");
|
||||
let read = disk_a
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read committed A");
|
||||
assert_eq!(read.data, fi.data, "warm={warm}");
|
||||
assert_eq!(
|
||||
tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
|
||||
.await
|
||||
.expect("B metadata"),
|
||||
b_before
|
||||
);
|
||||
assert!(b.path().join("target-bucket/staged/xl.meta").exists());
|
||||
assert!(ctx_a.namespace_commit_generation() > 0);
|
||||
assert_eq!(ctx_b.namespace_commit_generation(), 0);
|
||||
assert!(!ctx_a.namespace_commits_pending());
|
||||
assert!(!ctx_b.namespace_commits_pending());
|
||||
assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
for state in ["removed", "quarantined", "replaced"] {
|
||||
let replacement = new_disk(
|
||||
&disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("separate active Arc");
|
||||
let map = ctx.local_disk_map();
|
||||
let mut entries = map.write().await;
|
||||
match state {
|
||||
"removed" => {
|
||||
entries.remove(&endpoint);
|
||||
}
|
||||
"quarantined" => {
|
||||
entries.insert(endpoint.clone(), None);
|
||||
}
|
||||
_ => {
|
||||
entries.insert(endpoint.clone(), Some(replacement));
|
||||
}
|
||||
}
|
||||
drop(entries);
|
||||
assert!(
|
||||
matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)),
|
||||
"{state}"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let old_id = Uuid::new_v4();
|
||||
let old = target_disk(&ctx, root.path(), old_id).await;
|
||||
assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
|
||||
let replacement_id = Uuid::new_v4();
|
||||
let replacement = target_disk(&ctx, root.path(), replacement_id).await;
|
||||
assert!(!Arc::ptr_eq(&old, &replacement));
|
||||
assert!(matches!(
|
||||
local_disk_candidate(&ctx, &old_id.to_string()).await,
|
||||
Err(DiskError::DiskNotFound)
|
||||
));
|
||||
let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
|
||||
.await
|
||||
.expect("replacement UUID");
|
||||
assert!(Arc::ptr_eq(&candidate, &replacement));
|
||||
assert_eq!(verified, Some(replacement_id));
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let target = BootstrapLocalTarget::new(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"body");
|
||||
let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
|
||||
let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
|
||||
for invalid in [
|
||||
"victim",
|
||||
".rustfs.sys/../victim",
|
||||
".rustfs.sys/./tmp",
|
||||
".rustfs.sys/ .. /victim",
|
||||
".rustfs.sys\\..\\victim",
|
||||
".minio.sys/../victim",
|
||||
] {
|
||||
for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
|
||||
.await
|
||||
.is_err(),
|
||||
"src={src}, dst={dst}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.undo_local_write(
|
||||
&disk.endpoint().to_string(),
|
||||
invalid,
|
||||
"staged",
|
||||
fi.clone(),
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"{invalid}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
(".rustfs.sys/tmp", "staged"),
|
||||
&fi,
|
||||
(".rustfs.sys/tmp", "destination"),
|
||||
Some(Uuid::new_v4())
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("victim/staged/xl.meta"))
|
||||
.await
|
||||
.expect("user source"),
|
||||
user_before
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
|
||||
.await
|
||||
.expect("metadata source"),
|
||||
meta_before
|
||||
);
|
||||
assert!(!root.path().join("victim/destination").exists());
|
||||
assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
|
||||
for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
|
||||
seed_target(&disk, volume, "staged", fi.clone()).await;
|
||||
BootstrapLocalTarget::new(ctx.clone())
|
||||
.rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
|
||||
.await
|
||||
.expect("legitimate bootstrap metadata write");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read bootstrap result");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let sibling = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let (token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("real scanner token in A");
|
||||
let destination = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local disk")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&destination, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token),
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical rename entered"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
drop(rename);
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(!sibling.namespace_commits_pending());
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
|
||||
.await
|
||||
.is_err(),
|
||||
"real pending rename blocks another scanner publication"
|
||||
);
|
||||
assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
|
||||
let gate = ctx.data_movement_operation_gate();
|
||||
assert!(
|
||||
gate.clone().try_write_owned().is_err(),
|
||||
"physical operation still owns the scanner read guard"
|
||||
);
|
||||
drop(release_tx);
|
||||
let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
|
||||
.await
|
||||
.expect("physical tail must release scanner guard");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("namespace owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual late commit");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert!(ctx.namespace_commit_generation() >= 2);
|
||||
assert_eq!(sibling.namespace_commit_generation(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let other = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let other_store = super::super::tests::build_store_with_ctx(other);
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
|
||||
let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
|
||||
let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
|
||||
let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
|
||||
assert!(store.release_scanner_publication_lease(released).await);
|
||||
let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
|
||||
for token in [Uuid::new_v4(), foreign, released] {
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(valid)
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"expired real token"
|
||||
);
|
||||
let _ = other_store.release_scanner_publication_lease(foreign).await;
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
|
||||
.await
|
||||
.expect("source bytes"),
|
||||
before
|
||||
);
|
||||
assert!(!root.path().join("target-bucket/destination").exists());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
None,
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical entry"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded entry");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||
.await
|
||||
.expect("ordinary deadline remains enabled");
|
||||
assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
|
||||
drop(rename);
|
||||
assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late physical owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual timeout tail");
|
||||
assert_eq!(read.data, fi.data);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
|
||||
@@ -692,10 +692,15 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
let old_dir = v.object.as_ref().map(|v| v.data_dir).unwrap_or_default();
|
||||
// The version stays on disk while the purge replicates
|
||||
// (status PENDING/FAILED); its data dir must stay with
|
||||
// it. Returning the dir here made the disk layer delete
|
||||
// it, which turned every non-inline retained version
|
||||
// into an unreadable zombie: the purge state could never
|
||||
// be applied and the bucket could never be deleted.
|
||||
self.set_idx(i, v)?;
|
||||
|
||||
return Ok(old_dir);
|
||||
return Ok(None);
|
||||
}
|
||||
found_index = Some(i);
|
||||
}
|
||||
@@ -2702,6 +2707,58 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340: a version purge that still awaits
|
||||
/// the replication target keeps the object version on disk with a pending
|
||||
/// purge status. Its data dir must be retained with it; handing the dir
|
||||
/// back here made the disk layer delete it, leaving every non-inline
|
||||
/// retained version unreadable. The dir is released only once the purge
|
||||
/// completes and the version itself goes away.
|
||||
#[test]
|
||||
fn delete_version_pending_version_purge_retains_object_data_dir() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut fm = FileMeta::new();
|
||||
let mut fi = FileInfo::new("object", 2, 2);
|
||||
fi.version_id = Some(version_id);
|
||||
fi.data_dir = Some(data_dir);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fm.add_version(fi).unwrap();
|
||||
|
||||
let pending_purge = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
mark_deleted: true,
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=PENDING;".to_string()),
|
||||
purge_targets: version_purge_statuses_map("target=PENDING;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let freed = fm.delete_version(&pending_purge).unwrap();
|
||||
assert_eq!(freed, None, "a pending purge must not release the retained version's data dir");
|
||||
assert_eq!(fm.versions.len(), 1, "the version must stay until the purge replicates");
|
||||
let retained = fm
|
||||
.into_fileinfo("vol", "object", &version_id.to_string(), false, false, true)
|
||||
.unwrap();
|
||||
assert_eq!(retained.data_dir, Some(data_dir));
|
||||
assert_eq!(retained.version_purge_status(), VersionPurgeStatusType::Pending);
|
||||
|
||||
let completed_purge = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=COMPLETE;".to_string()),
|
||||
purge_targets: version_purge_statuses_map("target=COMPLETE;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let freed = fm.delete_version(&completed_purge).unwrap();
|
||||
assert_eq!(freed, Some(data_dir), "a completed purge removes the version and releases its data dir");
|
||||
assert!(fm.versions.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_accepts_delete_only_marker_and_free_version_paths() {
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
|
||||
+155
-19
@@ -76,6 +76,8 @@ struct HealTaskStatusPayload<'a> {
|
||||
min_seq: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
progress: Option<&'a HealProgress>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
outcome: Option<&'a super::outcome::HealTaskOutcome>,
|
||||
}
|
||||
|
||||
fn u64_is_zero(value: &u64) -> bool {
|
||||
@@ -87,17 +89,18 @@ fn encode_heal_task_status_payload(
|
||||
mut items: Vec<HealResultItem>,
|
||||
progress: Option<&HealProgress>,
|
||||
mut truncated: bool,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
sequence: (u64, u64),
|
||||
outcome: Option<&super::outcome::HealTaskOutcome>,
|
||||
) -> Result<(Vec<u8>, bool)> {
|
||||
loop {
|
||||
let data = serde_json::to_vec(&HealTaskStatusPayload {
|
||||
summary,
|
||||
items: &items,
|
||||
truncated,
|
||||
next_seq,
|
||||
min_seq,
|
||||
next_seq: sequence.0,
|
||||
min_seq: sequence.1,
|
||||
progress,
|
||||
outcome,
|
||||
})
|
||||
.map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
|
||||
if data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE {
|
||||
@@ -111,25 +114,21 @@ fn encode_heal_task_status_payload(
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
|
||||
if !truncated {
|
||||
return detail;
|
||||
}
|
||||
let truncation = "heal result items were truncated";
|
||||
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
|
||||
}
|
||||
|
||||
fn encode_heal_status_response(
|
||||
summary: &str,
|
||||
items: Vec<HealResultItem>,
|
||||
progress: Option<&HealProgress>,
|
||||
detail: Option<String>,
|
||||
truncated: bool,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
sequence: (u64, u64),
|
||||
outcome: Option<&super::outcome::HealTaskOutcome>,
|
||||
) -> Result<(Vec<u8>, Option<String>)> {
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?;
|
||||
Ok((data, heal_status_detail(detail, truncated)))
|
||||
let (summary, detail) = match outcome {
|
||||
Some(outcome) => outcome.legacy_status(summary, detail),
|
||||
None => (summary, detail),
|
||||
};
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, sequence, outcome)?;
|
||||
Ok((data, super::outcome::heal_status_detail(detail, truncated)))
|
||||
}
|
||||
|
||||
impl HealChannelProcessor {
|
||||
@@ -439,6 +438,7 @@ impl HealChannelProcessor {
|
||||
.await
|
||||
};
|
||||
|
||||
let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
|
||||
let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report {
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending | HealTaskStatus::Running,
|
||||
@@ -576,8 +576,15 @@ impl HealChannelProcessor {
|
||||
}
|
||||
};
|
||||
|
||||
let (data, detail) =
|
||||
encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?;
|
||||
let (data, detail) = encode_heal_status_response(
|
||||
&summary,
|
||||
items,
|
||||
progress.as_ref(),
|
||||
detail,
|
||||
truncated,
|
||||
(next_seq, min_seq),
|
||||
outcome.as_deref(),
|
||||
)?;
|
||||
|
||||
let response = HealChannelResponse {
|
||||
request_id: client_token,
|
||||
@@ -866,7 +873,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap();
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false, (0, 0), None).unwrap();
|
||||
|
||||
assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
|
||||
let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
|
||||
@@ -875,6 +882,135 @@ mod tests {
|
||||
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_fixture_matches_canonical_owner_and_preserves_legacy_terminals() {
|
||||
use crate::heal::outcome::*;
|
||||
let cases: serde_json::Value = serde_json::from_str(include_str!("../../../madmin/tests/fixtures/heal-outcome-v3.json"))
|
||||
.expect("shared client fixtures");
|
||||
for case in cases.as_array().expect("fixture cases") {
|
||||
if case.get("remoteResponse").is_some() {
|
||||
continue;
|
||||
}
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
let name = case["name"].as_str().expect("case name");
|
||||
if matches!(name, "unknown" | "completed_with_errors") {
|
||||
let disposition = if name == "unknown" {
|
||||
HealObjectDisposition::Unknown
|
||||
} else {
|
||||
outcome.attempt_failed();
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
}
|
||||
let abort = match name {
|
||||
"cancelled" => Some(HealAbortReason::Cancelled),
|
||||
"deadline" => Some(HealAbortReason::Deadline),
|
||||
"untraversable" => Some(HealAbortReason::Untraversable),
|
||||
_ => None,
|
||||
};
|
||||
outcome.finish(abort);
|
||||
let expected = &case["response"];
|
||||
let initial_detail = abort.map(|reason| {
|
||||
match reason {
|
||||
HealAbortReason::Cancelled => "heal task cancelled",
|
||||
HealAbortReason::Deadline => "heal task timed out",
|
||||
HealAbortReason::Untraversable => "heal listing is untraversable",
|
||||
}
|
||||
.to_string()
|
||||
});
|
||||
let (bytes, detail) = encode_heal_status_response(
|
||||
if abort.is_some() { "stopped" } else { "finished" },
|
||||
Vec::new(),
|
||||
None,
|
||||
initial_detail,
|
||||
true,
|
||||
(9, 4),
|
||||
Some(&outcome),
|
||||
)
|
||||
.expect("canonical owner encoding");
|
||||
let decoded: serde_json::Value = serde_json::from_slice(&bytes).expect("wire payload");
|
||||
assert_eq!(decoded["summary"], expected["summary"], "{name}");
|
||||
assert_eq!(detail.unwrap_or_default(), expected["detail"].as_str().expect("detail"), "{name}");
|
||||
assert_eq!(decoded["outcome"], expected["outcome"], "{name}");
|
||||
assert_eq!((decoded["next_seq"].as_u64(), decoded["min_seq"].as_u64()), (Some(9), Some(4)));
|
||||
assert!(decoded["outcome"].get("retainedObjectBytes").is_none());
|
||||
assert!(decoded["outcome"].get("untraversable").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_abort_cannot_be_hidden_by_a_finished_status() {
|
||||
use crate::heal::outcome::{HealAbortReason, HealTaskOutcome};
|
||||
for reason in [
|
||||
HealAbortReason::Cancelled,
|
||||
HealAbortReason::Deadline,
|
||||
HealAbortReason::Untraversable,
|
||||
] {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.finish(Some(reason));
|
||||
let (data, detail) = encode_heal_status_response("finished", Vec::new(), None, None, false, (0, 0), Some(&outcome))
|
||||
.expect("canonical abort adapter");
|
||||
let json: serde_json::Value = serde_json::from_slice(&data).expect("public state");
|
||||
assert_eq!(json["summary"], "stopped");
|
||||
assert_eq!(json["outcome"]["execution"]["state"], "aborted");
|
||||
assert!(detail.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_payload_bound_keeps_cumulative_outcome_and_cursors() {
|
||||
use crate::heal::outcome::*;
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
for index in 0..256 {
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: format!("object-{index}"),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
detail: Some("\"".repeat(1024)),
|
||||
});
|
||||
}
|
||||
let retained = outcome.objects.len();
|
||||
let items = vec![
|
||||
HealResultItem::default(),
|
||||
HealResultItem {
|
||||
detail: "x".repeat(MAX_HEAL_STATUS_PAYLOAD_SIZE + 1),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (bytes, detail) = encode_heal_status_response("running", items, None, None, false, (9, 4), Some(&outcome))
|
||||
.expect("bounded status with cumulative outcome");
|
||||
assert!(bytes.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
|
||||
let wire: serde_json::Value = serde_json::from_slice(&bytes).expect("bounded payload");
|
||||
assert_eq!(wire["items"].as_array().expect("items").len(), 1);
|
||||
assert_eq!(wire["truncated"], true);
|
||||
assert_eq!((wire["next_seq"].as_u64(), wire["min_seq"].as_u64()), (Some(9), Some(4)));
|
||||
assert_eq!(wire["outcome"]["counters"]["processed"], 256);
|
||||
assert_eq!(wire["outcome"]["counters"]["healed"], 0);
|
||||
assert_eq!(wire["outcome"]["objects"].as_array().expect("outcome window").len(), retained);
|
||||
assert!(retained < 256 && outcome.objects_truncated);
|
||||
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_response_preserves_all_admission_outcomes() {
|
||||
let cases = [
|
||||
|
||||
@@ -150,6 +150,7 @@ impl HealManager {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
&& !super::super::replacement_readiness::directory_backed_replacement_fallback_enabled()
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
|
||||
@@ -313,7 +313,6 @@ impl HealManager {
|
||||
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
@@ -375,11 +374,11 @@ impl HealManager {
|
||||
drop(stats);
|
||||
if terminal_completion {
|
||||
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
|
||||
if successful_completion {
|
||||
emit_mrf_repaired_events(notice_targets);
|
||||
} else {
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
// Neither task status nor the diagnostic outcome
|
||||
// window supplies a storage-owned repair receipt.
|
||||
// Release only the ingress lease for rediscovery;
|
||||
// preserve the producer's existing retry hints.
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -706,20 +705,6 @@ fn move_mrf_repair_notice_targets(
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
@@ -746,11 +731,8 @@ pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
|
||||
completed_heals.retain(|_, completed| {
|
||||
now.duration_since(completed.completed_at)
|
||||
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
completed_heals
|
||||
.retain(|_, completed| now.duration_since(completed.completed_at).unwrap_or_default() <= KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
|
||||
key.capacity()
|
||||
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
|
||||
|
||||
@@ -189,37 +189,104 @@ fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
|
||||
);
|
||||
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(entries.contains_key("ttl-boundary"));
|
||||
assert!(entries.contains_key("future"), "clock rollback must not expire a new completion");
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert!(entries.contains_key("future"));
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1) + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert!(entries.contains_key("future"), "the exact TTL boundary remains retained");
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(2) + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_clock_rollback_preserves_terminal_alias_queries() {
|
||||
let completed_at = SystemTime::now() + Duration::from_secs(3600);
|
||||
for status in [
|
||||
HealTaskStatus::Completed,
|
||||
HealTaskStatus::Failed {
|
||||
error: "fixture failure".to_string(),
|
||||
},
|
||||
HealTaskStatus::Cancelled,
|
||||
] {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut snapshot = completed_retention_fixture(completed_at);
|
||||
snapshot.status = status.clone();
|
||||
let expected_progress = snapshot.progress.clone();
|
||||
let snapshot = Arc::new(snapshot);
|
||||
{
|
||||
let mut completed = manager.completed_heals.lock().await;
|
||||
completed.insert("canonical".to_string(), Arc::clone(&snapshot));
|
||||
completed.insert("alias".to_string(), Arc::clone(&snapshot));
|
||||
}
|
||||
for token in ["canonical", "alias"] {
|
||||
let report = manager
|
||||
.get_task_report_since(token, Some(3))
|
||||
.await
|
||||
.expect("a clock rollback must retain terminal queries");
|
||||
assert_eq!(report.status, status);
|
||||
assert_eq!(report.progress, expected_progress);
|
||||
assert_eq!(report.result_items.len(), 1);
|
||||
assert_eq!((report.min_seq, report.next_seq), (3, 5));
|
||||
assert!(!report.result_items_truncated);
|
||||
}
|
||||
let mut completed = manager.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses_at(&mut completed, completed_at + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert_eq!(completed.len(), 2, "both tokens remain at the exact TTL boundary");
|
||||
prune_completed_heal_statuses_at(&mut completed, completed_at + KEEP_HEAL_TASK_STATUS_DURATION + Duration::from_nanos(1));
|
||||
assert!(completed.is_empty(), "both tokens expire after the TTL");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_clock_rollback_keeps_count_and_alias_eviction_bounded() {
|
||||
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(3600);
|
||||
let oldest = Arc::new(completed_retention_fixture(now + Duration::from_secs(1)));
|
||||
let mut entries = HashMap::from([
|
||||
("oldest".to_string(), Arc::clone(&oldest)),
|
||||
("oldest-alias".to_string(), oldest),
|
||||
]);
|
||||
for index in 2..=MAX_COMPLETED_HEAL_TOKENS {
|
||||
entries.insert(
|
||||
format!("task-{index}"),
|
||||
Arc::new(completed_retention_fixture(now + Duration::from_secs(2))),
|
||||
);
|
||||
}
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
|
||||
assert!(!entries.contains_key("oldest"));
|
||||
assert!(!entries.contains_key("oldest-alias"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_total_byte_cap_and_cap_plus_one() {
|
||||
let now = SystemTime::now();
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(now);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
for completed_at in [now, now + Duration::from_secs(1)] {
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(completed_at);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -877,6 +944,84 @@ fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSourc
|
||||
request
|
||||
}
|
||||
|
||||
fn scoped_object_request(bucket: &str, object: &str, pool_index: usize, set_index: usize) -> HealRequest {
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_bulkhead_starts_other_sets_and_retains_same_set_tail() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
{
|
||||
let mut config = manager.config.write().await;
|
||||
config.max_concurrent_heals = 2;
|
||||
config.max_concurrent_per_set = 1;
|
||||
config.set_bulkhead_enable = true;
|
||||
config.event_driven_scheduler_enable = false;
|
||||
config.mainline_throttle_enable = false;
|
||||
}
|
||||
|
||||
let first_set = scoped_object_request("scheduler-bulkhead-set-a-first", "object-a", 0, 1);
|
||||
let first_set_id = first_set.id.clone();
|
||||
let same_set_tail = scoped_object_request("scheduler-bulkhead-set-a-tail", "object-b", 0, 1);
|
||||
let same_set_tail_id = same_set_tail.id.clone();
|
||||
let other_set = scoped_object_request("scheduler-bulkhead-set-b", "object-c", 0, 2);
|
||||
let other_set_id = other_set.id.clone();
|
||||
|
||||
let first_hook = Arc::new(CompletedRetentionHook::default());
|
||||
let other_hook = Arc::new(CompletedRetentionHook::default());
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert("scheduler-bulkhead-set-a-first".to_string(), first_hook.clone());
|
||||
hooks.insert("scheduler-bulkhead-set-b".to_string(), other_hook.clone());
|
||||
}
|
||||
{
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.push(first_set), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(same_set_tail), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(other_set), QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), first_hook.started.notified())
|
||||
.await
|
||||
.expect("first set task should start");
|
||||
tokio::time::timeout(Duration::from_secs(5), other_hook.started.notified())
|
||||
.await
|
||||
.expect("other set task should start despite same-set tail");
|
||||
|
||||
assert_eq!(manager.get_active_task_count().await, 2);
|
||||
assert_eq!(manager.get_queue_length().await, 1);
|
||||
assert!(matches!(manager.get_task_status(&same_set_tail_id).await, Ok(HealTaskStatus::Pending)));
|
||||
{
|
||||
let active = manager.active_heals.lock().await;
|
||||
assert!(active.contains_key(&first_set_id));
|
||||
assert!(active.contains_key(&other_set_id));
|
||||
assert!(!active.contains_key(&same_set_tail_id));
|
||||
let counts = running_heal_set_counts(&active);
|
||||
assert_eq!(counts.get("pool_0_set_1"), Some(&1));
|
||||
assert_eq!(counts.get("pool_0_set_2"), Some(&1));
|
||||
}
|
||||
|
||||
manager.cancel_task(&first_set_id).await.expect("cancel first active task");
|
||||
manager.cancel_task(&other_set_id).await.expect("cancel other active task");
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|bucket, _| !bucket.starts_with("scheduler-bulkhead-"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_push_displacing_lower_priority_actually_enqueues_new_request() {
|
||||
// Regression for the release-build defect where the enqueue side effect lived inside
|
||||
@@ -3096,7 +3241,7 @@ async fn test_cancel_task_removes_queued_request() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mrf_repaired_notice_waits_for_successful_completion() {
|
||||
async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
|
||||
let bucket = "mrf-completion-success";
|
||||
let object = "object";
|
||||
let version_id = Some([9u8; 16]);
|
||||
@@ -3126,21 +3271,81 @@ async fn test_mrf_repaired_notice_waits_for_successful_completion() {
|
||||
);
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
for _ in 0..100 {
|
||||
let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
if !events.is_empty() {
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].object.as_ref(), object);
|
||||
assert_eq!(events[0].version_id, version_id);
|
||||
return;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let stats = manager.get_statistics().await;
|
||||
if stats.successful_tasks + stats.failed_tasks > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("successful MRF-owned heal should emit one repaired event");
|
||||
})
|
||||
.await
|
||||
.expect("scheduler completes the task");
|
||||
assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
|
||||
assert!(!lock_mrf_repair_notice_targets(&manager.mrf_repair_notice_targets).contains_key(&receipt.task_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mrf_repaired_notice_removed_on_queued_cancel_without_event() {
|
||||
async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() {
|
||||
for empty_window in [false, true] {
|
||||
let bucket = if empty_window {
|
||||
"mrf-empty-outcome"
|
||||
} else {
|
||||
"mrf-dry-run-outcome"
|
||||
};
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::new(
|
||||
if empty_window {
|
||||
HealType::Cluster
|
||||
} else {
|
||||
HealType::Object {
|
||||
bucket: bucket.to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
}
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run: !empty_window,
|
||||
recreate_missing: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt(request, Arc::from(bucket), Arc::from("object"), None)
|
||||
.await
|
||||
.expect("notice target registered");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let stats = manager.get_statistics().await;
|
||||
if stats.successful_tasks + stats.failed_tasks > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("scheduler completed");
|
||||
let report = manager.get_task_report(&receipt.task_id).await.expect("completed report");
|
||||
assert_eq!(report.status, HealTaskStatus::Completed);
|
||||
let outcome = report.outcome.expect("canonical outcome");
|
||||
if empty_window {
|
||||
assert!(outcome.objects.is_empty());
|
||||
} else {
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
crate::heal::outcome::HealObjectDisposition::DryRunObserved
|
||||
);
|
||||
}
|
||||
assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
|
||||
let bucket = "mrf-completion-cancel";
|
||||
let object = "object";
|
||||
let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
|
||||
@@ -25,12 +25,13 @@
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable because
|
||||
//! every producer keeps its own safety net: read-repair re-detects on the
|
||||
//! next failing read, and the scanner's corrupt-metadata branch leaves a
|
||||
//! pending-ledger entry behind even when its MRF intent is accepted
|
||||
//! (backlog#1894 axis A), so a lost intent is retried by the ledger rather
|
||||
//! than waiting for the failed-object TTL to re-scan the path.
|
||||
//! own CRC32. Neither ingress nor manager admission is a durable ownership
|
||||
//! receipt. The last flush window can be lost. Read-repair can rediscover a
|
||||
//! failed read; the scanner retains bounded, expiring retry hints. Partial
|
||||
//! writes also use a best-effort in-memory fast path, not a durable successor.
|
||||
//! These mechanisms must not be reported as verified repair completion.
|
||||
//! The partial-write caller's restart-survival requirement remains unmet by
|
||||
//! admission alone; a verified durable handoff is still required.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
|
||||
@@ -585,8 +586,8 @@ impl MrfRuntime {
|
||||
self.dirty = true;
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot. The scanner ledger is cleared later, when the
|
||||
// canonical heal task reaches a successful terminal completion.
|
||||
// smaller snapshot. This is not a durable successor receipt and
|
||||
// does not discharge the producer's existing retry hints.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
|
||||
@@ -123,6 +123,13 @@ pub struct CommittedSnapshot {
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SnapshotReadStats {
|
||||
file_reads: usize,
|
||||
bytes_read: usize,
|
||||
peak_file_bytes: usize,
|
||||
}
|
||||
|
||||
impl CommittedSnapshot {
|
||||
/// Persistent single-writer sequence, not a process UUID ordering.
|
||||
pub fn sequence(&self) -> u64 {
|
||||
@@ -162,6 +169,15 @@ pub enum RecoverySnapshot {
|
||||
}
|
||||
|
||||
async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
|
||||
read_bounded_with_stats(disk, path, limit, None).await
|
||||
}
|
||||
|
||||
async fn read_bounded_with_stats(
|
||||
disk: &EcstoreDiskStore,
|
||||
path: &str,
|
||||
limit: usize,
|
||||
mut stats: Option<&mut SnapshotReadStats>,
|
||||
) -> Result<Option<Vec<u8>>, SnapshotError> {
|
||||
let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(reader) => reader,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
|
||||
@@ -178,6 +194,11 @@ async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Resu
|
||||
if bytes.len() > limit {
|
||||
return Err(SnapshotError::TooLarge);
|
||||
}
|
||||
if let Some(stats) = stats.as_mut() {
|
||||
stats.file_reads += 1;
|
||||
stats.bytes_read = stats.bytes_read.checked_add(bytes.len()).ok_or(SnapshotError::TooLarge)?;
|
||||
stats.peak_file_bytes = stats.peak_file_bytes.max(bytes.len());
|
||||
}
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
|
||||
@@ -197,17 +218,26 @@ fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: Committe
|
||||
}
|
||||
|
||||
async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
|
||||
read_committed_with_stats(disks, limit, None).await
|
||||
}
|
||||
|
||||
async fn read_committed_with_stats(
|
||||
disks: &[EcstoreDiskStore],
|
||||
limit: usize,
|
||||
mut stats: Option<&mut SnapshotReadStats>,
|
||||
) -> Result<Option<CommittedSnapshot>, SnapshotError> {
|
||||
let mut selected = None;
|
||||
let mut damaged = None;
|
||||
let mut identities = HashMap::new();
|
||||
for disk in disks {
|
||||
for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
|
||||
let candidate = async {
|
||||
let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
|
||||
let Some(manifest) = read_bounded_with_stats(disk, manifest_path, MANIFEST_LEN, stats.as_deref_mut()).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let header = Manifest::decode(&manifest, limit)?;
|
||||
let payload = read_bounded(disk, payload_path, header.payload_len)
|
||||
let payload = read_bounded_with_stats(disk, payload_path, header.payload_len, stats.as_deref_mut())
|
||||
.await?
|
||||
.ok_or(SnapshotError::Corrupt)?;
|
||||
CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
|
||||
@@ -492,6 +522,69 @@ mod tests {
|
||||
assert_eq!(recovered.manifest.sequence, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_reader_reopens_previous_anchor_across_publication_boundaries() {
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
let boundaries = [
|
||||
("payload-only", next.clone(), None),
|
||||
("torn-manifest", next.clone(), Some(manifest(owner, 2, &next)[..20].to_vec())),
|
||||
("stale-payload", old.clone(), Some(manifest(owner, 2, &next))),
|
||||
];
|
||||
for (case, successor_payload, successor_manifest) in boundaries {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
commit(&store, 0, owner, 1, &old).await;
|
||||
install(&store, PAYLOAD_PATHS[1], &successor_payload).await;
|
||||
if let Some(manifest) = &successor_manifest {
|
||||
install(&store, MANIFEST_PATHS[1], manifest).await;
|
||||
}
|
||||
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_committed(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{case}: old anchor must remain readable after reopen: {error:?}"))
|
||||
.unwrap_or_else(|| panic!("{case}: previous committed anchor missing after reopen"));
|
||||
assert_eq!(recovered.manifest.sequence, 1, "{case}: successor must not become authoritative");
|
||||
assert_eq!(recovered.payload, old, "{case}: previous payload must survive");
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload retained")
|
||||
.as_ref(),
|
||||
old.as_slice(),
|
||||
"{case}: previous payload bytes changed"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &old).as_slice(),
|
||||
"{case}: previous manifest bytes changed"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1])
|
||||
.await
|
||||
.expect("successor payload retained")
|
||||
.as_ref(),
|
||||
successor_payload.as_slice(),
|
||||
"{case}: successor evidence changed"
|
||||
);
|
||||
if let Some(manifest) = &successor_manifest {
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1])
|
||||
.await
|
||||
.expect("successor manifest retained")
|
||||
.as_ref(),
|
||||
manifest.as_slice(),
|
||||
"{case}: successor manifest evidence changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_manifest_cas_cannot_replace_committed_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
@@ -571,6 +664,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_reader_resource_bounds_are_measured() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = [payload("old-0"), payload("old-1")].concat();
|
||||
let new = [payload("new-0"), payload("new-1"), payload("new-2")].concat();
|
||||
commit(&first, 0, owner, 1, &old).await;
|
||||
commit(&second, 1, owner, 2, &new).await;
|
||||
|
||||
let mut stats = SnapshotReadStats::default();
|
||||
let recovered = read_committed_with_stats(&[first, second], 4096, Some(&mut stats))
|
||||
.await
|
||||
.expect("read committed replicas")
|
||||
.expect("committed snapshot");
|
||||
|
||||
assert_eq!(recovered.sequence(), 2);
|
||||
assert_eq!(recovered.payload(), new.as_slice());
|
||||
assert_eq!(recovered.manifest.payload_len, new.len());
|
||||
assert_eq!(stats.file_reads, 4, "only committed manifests and their payloads are materialized");
|
||||
assert_eq!(stats.bytes_read, (MANIFEST_LEN * 2) + old.len() + new.len());
|
||||
assert_eq!(
|
||||
stats.peak_file_bytes,
|
||||
new.len().max(MANIFEST_LEN),
|
||||
"reader peak allocation remains bounded by one manifest or payload file"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
|
||||
let scoped = |set_index| {
|
||||
|
||||
+179
-11
@@ -15,6 +15,7 @@
|
||||
//! Execution results are separate from repair responsibility. A legacy
|
||||
//! successful storage call supplies no authoritative repair receipt.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::VecDeque, time::SystemTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -22,14 +23,16 @@ const MAX_OUTCOME_ITEMS: usize = 128;
|
||||
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
|
||||
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HealObjectKind {
|
||||
Object,
|
||||
Metadata,
|
||||
Decode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealObjectIdentity {
|
||||
pub kind: HealObjectKind,
|
||||
pub bucket: String,
|
||||
@@ -41,7 +44,8 @@ pub struct HealObjectIdentity {
|
||||
pub set_index: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HealDeferredReason {
|
||||
DanglingDeleteGrace,
|
||||
TransientUsageCache,
|
||||
@@ -49,14 +53,21 @@ pub enum HealDeferredReason {
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HealFailureClass {
|
||||
Recoverable,
|
||||
RetryExhausted,
|
||||
Permanent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(
|
||||
tag = "state",
|
||||
content = "details",
|
||||
rename_all = "snake_case",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
pub enum HealObjectDisposition {
|
||||
/// The legacy storage response does not prove the requested check or commit.
|
||||
Unknown,
|
||||
@@ -72,7 +83,8 @@ pub enum HealObjectDisposition {
|
||||
DryRunObserved,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealObjectOutcome {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
@@ -89,7 +101,8 @@ impl HealObjectOutcome {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HealTraversalCoverage {
|
||||
#[default]
|
||||
Unknown,
|
||||
@@ -97,14 +110,16 @@ pub enum HealTraversalCoverage {
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum HealAbortReason {
|
||||
Cancelled,
|
||||
Deadline,
|
||||
Untraversable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "state", content = "reason", rename_all = "snake_case")]
|
||||
pub enum HealExecutionOutcome {
|
||||
#[default]
|
||||
Pending,
|
||||
@@ -114,7 +129,8 @@ pub enum HealExecutionOutcome {
|
||||
Aborted(HealAbortReason),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealOutcomeCounters {
|
||||
pub processed: u64,
|
||||
pub healed: u64,
|
||||
@@ -127,7 +143,8 @@ pub struct HealOutcomeCounters {
|
||||
pub overflowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealTaskOutcome {
|
||||
pub execution: HealExecutionOutcome,
|
||||
pub coverage: HealTraversalCoverage,
|
||||
@@ -135,11 +152,99 @@ pub struct HealTaskOutcome {
|
||||
/// A bounded diagnostic window, not a complete responsibility ledger.
|
||||
pub objects: VecDeque<HealObjectOutcome>,
|
||||
pub objects_truncated: bool,
|
||||
#[serde(skip)]
|
||||
retained_object_bytes: usize,
|
||||
#[serde(skip)]
|
||||
untraversable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum HealOutcomeWireError {
|
||||
#[error("heal outcome is missing execution or counters")]
|
||||
MissingFields,
|
||||
#[error("heal outcome has invalid or unsupported execution fields")]
|
||||
InvalidFields(#[from] serde_json::Error),
|
||||
#[error("finished heal summary contradicts its canonical outcome")]
|
||||
ContradictoryCompletion,
|
||||
}
|
||||
|
||||
/// Reconcile a peer's successful legacy summary using the canonical owner types.
|
||||
/// A running retry may legitimately retain the preceding attempt's outcome.
|
||||
pub fn legacy_wire_status<'a>(
|
||||
summary: &'a str,
|
||||
wire: &serde_json::Value,
|
||||
truncated: bool,
|
||||
) -> Result<(&'a str, Option<String>), HealOutcomeWireError> {
|
||||
if summary != "finished" {
|
||||
return Ok((summary, None));
|
||||
}
|
||||
let execution = HealExecutionOutcome::deserialize(wire.get("execution").ok_or(HealOutcomeWireError::MissingFields)?)?;
|
||||
let counters = HealOutcomeCounters::deserialize(wire.get("counters").ok_or(HealOutcomeWireError::MissingFields)?)?;
|
||||
if matches!(execution, HealExecutionOutcome::Pending | HealExecutionOutcome::Running)
|
||||
|| (execution == HealExecutionOutcome::Completed && counters.failed > 0)
|
||||
{
|
||||
return Err(HealOutcomeWireError::ContradictoryCompletion);
|
||||
}
|
||||
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
|
||||
Ok((
|
||||
adapted,
|
||||
if adapted != summary {
|
||||
heal_status_detail(detail, truncated)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
|
||||
if !truncated {
|
||||
return detail;
|
||||
}
|
||||
let truncation = "heal result items were truncated";
|
||||
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
|
||||
}
|
||||
|
||||
fn legacy_execution_status<'a>(
|
||||
summary: &'a str,
|
||||
detail: Option<String>,
|
||||
execution: HealExecutionOutcome,
|
||||
counters: &HealOutcomeCounters,
|
||||
) -> (&'a str, Option<String>) {
|
||||
if summary != "finished" {
|
||||
return (summary, detail);
|
||||
}
|
||||
match execution {
|
||||
HealExecutionOutcome::CompletedWithErrors => (
|
||||
"stopped",
|
||||
Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
|
||||
),
|
||||
HealExecutionOutcome::Aborted(reason) => {
|
||||
let reason = match reason {
|
||||
HealAbortReason::Cancelled => "cancelled",
|
||||
HealAbortReason::Deadline => "timed out",
|
||||
HealAbortReason::Untraversable => "untraversable",
|
||||
};
|
||||
("stopped", Some(format!("heal task {reason}")))
|
||||
}
|
||||
HealExecutionOutcome::Completed if counters.unknown > 0 => (
|
||||
summary,
|
||||
Some(format!(
|
||||
"heal traversal completed; authoritative storage proof is unavailable for {} objects",
|
||||
counters.unknown
|
||||
)),
|
||||
),
|
||||
HealExecutionOutcome::Pending | HealExecutionOutcome::Running => {
|
||||
("running", Some("heal execution has not reached a terminal outcome".to_string()))
|
||||
}
|
||||
HealExecutionOutcome::Completed => (summary, detail),
|
||||
}
|
||||
}
|
||||
|
||||
impl HealTaskOutcome {
|
||||
pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
|
||||
legacy_execution_status(summary, detail, self.execution, &self.counters)
|
||||
}
|
||||
|
||||
pub(crate) fn start(&mut self) {
|
||||
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
self.execution = HealExecutionOutcome::Running;
|
||||
@@ -292,6 +397,69 @@ mod canonical_outcome_tests {
|
||||
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_serialization_keeps_unverified_dispositions_and_window_bounds() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for disposition in [
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
},
|
||||
HealObjectDisposition::DryRunObserved,
|
||||
] {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
outcome.finish(None);
|
||||
let wire = serde_json::to_value(&outcome).expect("canonical wire view");
|
||||
assert_eq!(wire["execution"]["state"], "completed");
|
||||
assert_eq!(wire["counters"]["healed"], 0);
|
||||
assert_eq!(wire["counters"]["skipped"], 3);
|
||||
assert_eq!(wire["objects"][1]["disposition"]["details"]["reason"], "dangling_delete_grace");
|
||||
assert!(wire["objects"][1]["identity"]["bucketIncarnationId"].is_null());
|
||||
for _ in 0..MAX_OUTCOME_ITEMS + 1 {
|
||||
let mut result = item(HealObjectDisposition::Unknown);
|
||||
result.detail = Some("\"".repeat(MAX_OUTCOME_DETAIL_BYTES));
|
||||
outcome.record(result);
|
||||
}
|
||||
let bytes = serde_json::to_vec(&outcome).expect("bounded canonical samples");
|
||||
assert!(
|
||||
bytes.len() < 8 * MAX_OUTCOME_BYTES,
|
||||
"JSON escaping remains bounded independently of object count"
|
||||
);
|
||||
assert!(outcome.objects_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_wire_consistency_rejects_unknown_success_without_rejecting_extensions() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.finish(None);
|
||||
let mut wire = serde_json::to_value(&outcome).expect("canonical snapshot");
|
||||
wire["execution"]["futureField"] = serde_json::json!({"new": true});
|
||||
wire["counters"]["futureCounter"] = serde_json::json!(42);
|
||||
assert_eq!(
|
||||
legacy_wire_status("finished", &wire, false).expect("unknown extension fields"),
|
||||
("finished", None)
|
||||
);
|
||||
wire["execution"]["state"] = serde_json::json!("future_execution");
|
||||
assert!(legacy_wire_status("finished", &wire, false).is_err());
|
||||
assert_eq!(
|
||||
legacy_wire_status("running", &wire, false).expect("unknown nonterminal outcome"),
|
||||
("running", None)
|
||||
);
|
||||
wire["execution"] = serde_json::json!({"state":"completed"});
|
||||
wire["counters"]["failed"] = serde_json::json!(1);
|
||||
assert!(matches!(
|
||||
legacy_wire_status("finished", &wire, false),
|
||||
Err(HealOutcomeWireError::ContradictoryCompletion)
|
||||
));
|
||||
wire.as_object_mut().expect("object").remove("execution");
|
||||
assert!(matches!(
|
||||
legacy_wire_status("finished", &wire, false),
|
||||
Err(HealOutcomeWireError::MissingFields)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
|
||||
@@ -18,6 +18,25 @@ use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
};
|
||||
|
||||
/// Whether automatic replacement may fall back to the set-wide format heal when
|
||||
/// a target cannot pass the independent-mount admission.
|
||||
///
|
||||
/// Directory-backed deployments already declare, through
|
||||
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
|
||||
/// directories sharing a device with the host root. Those endpoints can never
|
||||
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
|
||||
/// runtime-wiped or replaced directory disk would stay deferred forever. The
|
||||
/// admission check itself is never bypassed; the fallback only routes the heal
|
||||
/// through the ordinary format path that formats every unformatted disk in the
|
||||
/// set, which is exactly what the pre-admission `heal_disk` path did.
|
||||
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool_with_aliases(
|
||||
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||
&[rustfs_config::ENV_MINIO_CI],
|
||||
rustfs_config::DEFAULT_UNSAFE_BYPASS_DISK_CHECK,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
}
|
||||
@@ -184,12 +203,47 @@ mod tests {
|
||||
assert!(endpoint.is_local);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_backed_fallback_is_off_by_default() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
|| assert!(!directory_backed_replacement_fallback_enabled()),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_backed_fallback_follows_the_disk_check_bypass() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
|| assert!(directory_backed_replacement_fallback_enabled()),
|
||||
);
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("false")),
|
||||
(rustfs_config::ENV_MINIO_CI, Some("true")),
|
||||
],
|
||||
|| {
|
||||
assert!(
|
||||
!directory_backed_replacement_fallback_enabled(),
|
||||
"the canonical key must win over the alias"
|
||||
)
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_environment_cannot_bypass_mount_admission() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_TEST_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")),
|
||||
("RUSTFS_E2E_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")),
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
],
|
||||
async {
|
||||
let temp = TempDir::new().expect("temporary replacement root should be created");
|
||||
|
||||
@@ -14,8 +14,107 @@
|
||||
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
|
||||
use super::*;
|
||||
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
|
||||
use crate::heal::storage::HealListItem;
|
||||
use crate::heal::utils::format_set_disk_id;
|
||||
|
||||
const MAX_DEFERRED_OBJECTS: usize = 256;
|
||||
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
|
||||
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
|
||||
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
|
||||
|
||||
struct DeferredObject {
|
||||
name: String,
|
||||
version_id: Option<String>,
|
||||
attempt: u32,
|
||||
page: u64,
|
||||
first_failure: Option<tokio::time::Instant>,
|
||||
due: tokio::time::Instant,
|
||||
}
|
||||
|
||||
impl DeferredObject {
|
||||
fn new(item: HealListItem, page: u64) -> Self {
|
||||
Self {
|
||||
name: item.name,
|
||||
version_id: item.version_id,
|
||||
attempt: 0,
|
||||
page,
|
||||
first_failure: None,
|
||||
due: tokio::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn payload_bytes(&self) -> usize {
|
||||
self.name
|
||||
.capacity()
|
||||
.saturating_add(self.version_id.as_ref().map_or(0, String::capacity))
|
||||
}
|
||||
|
||||
fn expired(&self) -> bool {
|
||||
self.first_failure.is_some_and(|first| first.elapsed() >= MAX_DEFERRED_AGE)
|
||||
}
|
||||
|
||||
fn defer(&mut self, delay: Duration) {
|
||||
let now = tokio::time::Instant::now();
|
||||
let first = *self.first_failure.get_or_insert(now);
|
||||
self.attempt += 1;
|
||||
self.due = (now + delay).min(first + MAX_DEFERRED_AGE);
|
||||
}
|
||||
}
|
||||
|
||||
// Only failed identities are retained. The current listing page remains owned
|
||||
// by the caller; capacity pressure stops fetching, never discards that page.
|
||||
struct DeferredWindow {
|
||||
objects: VecDeque<DeferredObject>,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for DeferredWindow {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
objects: VecDeque::new(),
|
||||
// Charge every possible slot up front, including spare capacity.
|
||||
bytes: MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DeferredWindow {
|
||||
fn push(&mut self, item: DeferredObject) -> std::result::Result<(), DeferredObject> {
|
||||
let bytes = item.payload_bytes();
|
||||
if self.objects.len() >= MAX_DEFERRED_OBJECTS || bytes > MAX_DEFERRED_BYTES.saturating_sub(self.bytes) {
|
||||
return Err(item);
|
||||
}
|
||||
self.bytes += bytes;
|
||||
self.objects.push_back(item);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pop_due(&mut self) -> Option<DeferredObject> {
|
||||
let now = tokio::time::Instant::now();
|
||||
let index = self.objects.iter().position(|item| item.due <= now)?;
|
||||
let item = self.objects.remove(index)?;
|
||||
self.bytes -= item.payload_bytes();
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn next_due(&self) -> Option<tokio::time::Instant> {
|
||||
self.objects.iter().map(|item| item.due).min()
|
||||
}
|
||||
|
||||
fn can_advance(&self, page: u64) -> bool {
|
||||
self.objects.len() < MAX_DEFERRED_OBJECTS
|
||||
&& self.bytes < MAX_DEFERRED_BYTES
|
||||
&& self
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| page.saturating_sub(item.page) < MAX_DEFERRED_FORWARD_PAGES)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/deferred_retry_window.rs"]
|
||||
mod deferred_retry_window;
|
||||
|
||||
fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Option<Error> {
|
||||
if opts.dry_run || !opts.recreate {
|
||||
return None;
|
||||
@@ -305,83 +404,122 @@ impl HealTask {
|
||||
|
||||
for (set_disk_id, heal_opts) in listing_scopes {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let mut listing_attempt = 0;
|
||||
let (objects, next_token, is_truncated) = loop {
|
||||
let mut deferred = DeferredWindow::default();
|
||||
let mut inline_retry: Option<DeferredObject> = None;
|
||||
let mut page_number = 0_u64;
|
||||
let mut aborted_progress_unknown = false;
|
||||
let mut pending = Vec::<HealListItem>::new().into_iter();
|
||||
let mut listing_finished = false;
|
||||
let mut listing_attempt = 0;
|
||||
let mut listing_due = tokio::time::Instant::now();
|
||||
let scope_result: Result<()> = async {
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none()
|
||||
{
|
||||
break;
|
||||
}
|
||||
self.pace_mainline().await?;
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok(page) => break page,
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
// Listing and object retries share this safe boundary. A
|
||||
// failed listing never hides an already-due object retry.
|
||||
let item = deferred.pop_due().or_else(|| {
|
||||
if inline_retry
|
||||
.as_ref()
|
||||
.is_some_and(|item| item.due <= tokio::time::Instant::now())
|
||||
{
|
||||
inline_retry.take()
|
||||
} else if inline_retry.is_none() {
|
||||
pending.next().map(|item| DeferredObject::new(item, page_number))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
let Some(mut item) = item else {
|
||||
let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
|
||||
if can_list && listing_due <= tokio::time::Instant::now() {
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok((objects, next_token, is_truncated)) => {
|
||||
page_number = page_number.saturating_add(1);
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
listing_finished = continuation_token.is_none();
|
||||
listing_attempt = 0;
|
||||
listing_due = tokio::time::Instant::now();
|
||||
pending = objects.into_iter();
|
||||
}
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
listing_due =
|
||||
tokio::time::Instant::now() + self.bucket_object_retry_delay(listing_attempt);
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
let due = deferred
|
||||
.next_due()
|
||||
.into_iter()
|
||||
.chain(inline_retry.as_ref().map(|item| item.due))
|
||||
.chain(can_list.then_some(listing_due))
|
||||
.min();
|
||||
if let Some(due) = due {
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await;
|
||||
tokio::time::sleep_until(due).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let retry_attempt = item.attempt;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
}
|
||||
};
|
||||
|
||||
let mut pending = objects;
|
||||
let mut retry_attempt = 0_u32;
|
||||
while !pending.is_empty() {
|
||||
if retry_attempt > 0 {
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let mut retry = Vec::with_capacity(pending.len());
|
||||
for item in pending {
|
||||
self.check_control_flags().await?;
|
||||
self.pace_mainline().await?;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
}
|
||||
|
||||
let mut terminal_outcome = true;
|
||||
let error = match self
|
||||
let mut terminal_outcome = true;
|
||||
let age_exhausted = item.expired();
|
||||
let error = if age_exhausted {
|
||||
Some(Error::other("heal object retry age exhausted"))
|
||||
} else {
|
||||
match self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
@@ -414,152 +552,188 @@ impl HealTask {
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
return Err(err);
|
||||
}
|
||||
_ => self.outcome.write().await.attempt_failed(),
|
||||
}
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "dangling_delete_grace_skip",
|
||||
error = %err,
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
return Err(err);
|
||||
}
|
||||
_ if !age_exhausted => self.outcome.write().await.attempt_failed(),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if terminal_outcome {
|
||||
telemetry_unknown |= !increment_counter(&mut scanned);
|
||||
}
|
||||
|
||||
if !terminal_outcome {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "dangling_delete_grace_skip",
|
||||
error = %err,
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if !age_exhausted && err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
item.defer(self.bucket_object_retry_delay(retry_attempt + 1));
|
||||
if let Err(item) = deferred.push(item) {
|
||||
inline_retry = Some(item);
|
||||
}
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if age_exhausted || err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
pending = retry;
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
}
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
if terminal_outcome {
|
||||
telemetry_unknown |= !increment_counter(&mut scanned);
|
||||
}
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated without a continuation token is a compatibility EOF.
|
||||
break;
|
||||
if !terminal_outcome {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = scope_result {
|
||||
let disposition = match error {
|
||||
Error::TaskCancelled => HealObjectDisposition::Cancelled,
|
||||
Error::TaskTimeout => HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
},
|
||||
_ => HealObjectDisposition::Unknown,
|
||||
};
|
||||
// Only attempted identities have terminal outcomes. Unstarted
|
||||
// page tails remain unprocessed under the task's partial coverage.
|
||||
// No detached sleepers survive abort.
|
||||
for item in deferred.objects.into_iter().chain(inline_retry) {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity: self.outcome_identity(
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
heal_opts.pool,
|
||||
heal_opts.set,
|
||||
),
|
||||
disposition: disposition.clone(),
|
||||
detail: None,
|
||||
});
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
}
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if aborted_progress_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,36 @@ impl HealTask {
|
||||
progress.update_stage(0, 4);
|
||||
}
|
||||
|
||||
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
let mut is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
if is_auto_replacement
|
||||
&& crate::heal::replacement_readiness::directory_backed_replacement_fallback_enabled()
|
||||
&& self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
// Directory-backed endpoints cannot pass replacement admission; the
|
||||
// operator opted out of disk checks, so heal the set the way the
|
||||
// pre-admission `heal_disk` path did instead of deferring forever.
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "replacement_admission",
|
||||
result = "directory_backed_fallback",
|
||||
target_count = self.heal_endpoints.len(),
|
||||
"Heal erasure set falls back to set-wide format heal for a replacement target that is not an independently mounted disk"
|
||||
);
|
||||
is_auto_replacement = false;
|
||||
}
|
||||
let replacement_targets = if is_auto_replacement {
|
||||
self.heal_endpoints.clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
let mut requested_targets = self.heal_endpoints.clone();
|
||||
requested_targets.sort_unstable();
|
||||
@@ -421,7 +450,7 @@ impl HealTask {
|
||||
heal_opts,
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone())
|
||||
.with_mainline_pacer(self.mainline_pacer.clone());
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
|
||||
mod deferred_retry;
|
||||
|
||||
mod canonical_outcome {
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
@@ -478,6 +480,95 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
);
|
||||
}
|
||||
|
||||
fn directory_backed_replacement_request() -> HealRequest {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = vec!["/data/disk0".to_string()];
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_backed_replacement_falls_back_to_set_format_when_disk_checks_are_bypassed() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
async {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(false),
|
||||
global_format_ok_endpoints: Mutex::new(vec!["/data/disk0".to_string()]),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(directory_backed_replacement_request(), storage.clone());
|
||||
|
||||
// The mock has no local disk behind "/data/disk0", so the run stops at
|
||||
// the healing-marker step that follows the format stage, exactly like
|
||||
// `automatic_replacement_uses_target_scoped_format`. The assertions
|
||||
// below pin which format path ran before that point.
|
||||
let err = task.execute().await.expect_err("the mock has no local healing marker target");
|
||||
assert!(
|
||||
err.to_string().contains("healing marker target is unavailable"),
|
||||
"the fallback must reach the post-format marker step, got: {err}"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
*storage.global_format_calls.lock().unwrap(),
|
||||
1,
|
||||
"the fallback must run exactly one set-wide format heal"
|
||||
);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"the fallback must not run the target-scoped replacement format"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_backed_replacement_stays_fail_closed_without_disk_check_bypass() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
async {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(false),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(directory_backed_replacement_request(), storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect_err("an inadmissible replacement target must keep failing closed");
|
||||
|
||||
assert_eq!(
|
||||
*storage.global_format_calls.lock().unwrap(),
|
||||
0,
|
||||
"fail-closed admission must not format the set"
|
||||
);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"fail-closed admission must not format the target"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_persists_intent_before_format() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -939,6 +1030,10 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
retry_test_pages: Option<Vec<Vec<HealListItem>>>,
|
||||
retry_test_delays: HashMap<String, Duration>,
|
||||
retry_test_listing_delays: Mutex<VecDeque<Duration>>,
|
||||
retry_test_events: Mutex<Vec<String>>,
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
fail_second_listing_page: bool,
|
||||
@@ -956,6 +1051,8 @@ struct MockStorage {
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
/// Endpoints the set-wide format mock reports as freshly formatted (`state == "ok"`).
|
||||
global_format_ok_endpoints: Mutex<Vec<String>>,
|
||||
replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
|
||||
replacement_target_identities_ready: Mutex<bool>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<crate::heal::resume::ReplacementTargetIdentity>>>,
|
||||
@@ -1109,6 +1206,7 @@ fn replacement_identity(
|
||||
}
|
||||
|
||||
enum MockHealObjectOutcome {
|
||||
RetryableLock,
|
||||
OkWithOtherError(&'static str),
|
||||
ErrOther(&'static str),
|
||||
DanglingGraceDeferred,
|
||||
@@ -1213,6 +1311,10 @@ impl HealStorageAPI for MockStorage {
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.heal_object_calls.lock().unwrap().push(object.to_string());
|
||||
self.retry_test_events.lock().expect("events").push(format!("heal:{object}"));
|
||||
if let Some(delay) = self.retry_test_delays.get(object) {
|
||||
tokio::time::sleep(*delay).await;
|
||||
}
|
||||
self.heal_object_version_ids
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -1241,6 +1343,13 @@ impl HealStorageAPI for MockStorage {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableLock => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
|
||||
resource: object.to_string(),
|
||||
owner: "competing-writer".to_string(),
|
||||
}))),
|
||||
)),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
@@ -1266,6 +1375,13 @@ impl HealStorageAPI for MockStorage {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableLock => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
|
||||
resource: object.to_string(),
|
||||
owner: "competing-writer".to_string(),
|
||||
}))),
|
||||
)),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
@@ -1313,10 +1429,26 @@ impl HealStorageAPI for MockStorage {
|
||||
return Err(error);
|
||||
}
|
||||
let no_heal_required = *self.format_no_heal_required.lock().unwrap();
|
||||
let result = HealResultItem {
|
||||
after: Infos {
|
||||
drives: self
|
||||
.global_format_ok_endpoints
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|endpoint| HealDriveInfo {
|
||||
endpoint: endpoint.clone(),
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
if no_heal_required {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
|
||||
Ok((result, Some(Error::Storage(EcstoreError::NoHealRequired))))
|
||||
} else {
|
||||
Ok((HealResultItem::default(), None))
|
||||
Ok((result, None))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1361,6 +1493,19 @@ impl HealStorageAPI for MockStorage {
|
||||
.lock()
|
||||
.expect("listing tokens")
|
||||
.push(continuation_token.map(ToOwned::to_owned));
|
||||
self.retry_test_events
|
||||
.lock()
|
||||
.expect("events")
|
||||
.push(format!("list:{}", continuation_token.unwrap_or("first")));
|
||||
let delay = self.retry_test_listing_delays.lock().expect("listing delays").pop_front();
|
||||
if let Some(delay) = delay {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if let Some(pages) = &self.retry_test_pages {
|
||||
let page = continuation_token.map_or(0, |token| token.parse::<usize>().expect("test page token"));
|
||||
let next = (page + 1 < pages.len()).then(|| (page + 1).to_string());
|
||||
return Ok((pages[page].clone(), next.clone(), next.is_some()));
|
||||
}
|
||||
if let Some(remaining) = self
|
||||
.recoverable_second_page_failures
|
||||
.lock()
|
||||
|
||||
@@ -0,0 +1,487 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
|
||||
HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
)
|
||||
}
|
||||
|
||||
fn pages_storage(pages: &[&[&str]]) -> MockStorage {
|
||||
MockStorage {
|
||||
retry_test_pages: Some(
|
||||
pages
|
||||
.iter()
|
||||
.map(|page| page.iter().map(|name| heal_item(name)).collect())
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn fail_once(storage: &MockStorage, name: &str) {
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), VecDeque::from([MockHealObjectOutcome::RetryableLock]));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])),
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([
|
||||
MockHealObjectOutcome::RetryableSlowDown,
|
||||
MockHealObjectOutcome::RetryableSlowDown,
|
||||
]),
|
||||
);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(30_500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "heal:object-a"]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
);
|
||||
execution.await.expect_err("age exhausted object must remain a batch failure");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
[
|
||||
"list:first",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-b"
|
||||
]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(31_500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second"]
|
||||
);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 1);
|
||||
execution.await.expect_err("age exhaustion remains a failure");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unprocessed() {
|
||||
for cancel in [true, false] {
|
||||
let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
|
||||
let mut page: Vec<HealListItem> = names.iter().map(|name| heal_item(name)).collect();
|
||||
page[256].version_id = Some("inline-version".to_string());
|
||||
let storage = Arc::new(MockStorage {
|
||||
retry_test_pages: Some(vec![page, vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
});
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
}
|
||||
let mut task = bucket_task(storage.clone());
|
||||
if !cancel {
|
||||
task.options.timeout = Some(Duration::from_secs(1));
|
||||
}
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
if cancel {
|
||||
task.cancel().await.expect("cancel");
|
||||
}
|
||||
let result = execution.await;
|
||||
assert!(matches!(
|
||||
(&result, cancel),
|
||||
(Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
|
||||
));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(
|
||||
outcome.counters.processed,
|
||||
outcome.counters.skipped,
|
||||
outcome.counters.failed,
|
||||
outcome.counters.healed
|
||||
),
|
||||
(257, 257, 0, 0)
|
||||
);
|
||||
assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
outcome.execution,
|
||||
crate::heal::outcome::HealExecutionOutcome::Aborted(if cancel {
|
||||
HealAbortReason::Cancelled
|
||||
} else {
|
||||
HealAbortReason::Deadline
|
||||
})
|
||||
);
|
||||
let inline: Vec<_> = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "blocked-256")
|
||||
.collect();
|
||||
assert_eq!(inline.len(), 1);
|
||||
assert_eq!(inline[0].identity.version_id.as_deref(), Some("inline-version"));
|
||||
assert_eq!(
|
||||
inline[0].disposition,
|
||||
if cancel {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
}
|
||||
);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(
|
||||
(
|
||||
progress.objects_scanned,
|
||||
progress.skipped_objects,
|
||||
progress.objects_failed,
|
||||
progress.objects_healed
|
||||
),
|
||||
(257, 257, 0, 0)
|
||||
);
|
||||
assert!(
|
||||
!outcome
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.identity.object == "blocked-257" || item.identity.object == "healthy")
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
}
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
|
||||
execution.await.expect_err("all four objects exhaust retries");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(4, 4, 16)
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn listing_retry_keeps_cursor_and_does_not_replay_successful_objects() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect("both retries complete");
|
||||
assert_eq!(
|
||||
storage.listing_tokens.lock().expect("tokens").as_slice(),
|
||||
[None, Some("second".to_string()), Some("second".to_string())]
|
||||
);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-a", "object-b"]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn typed_lock_contention_allows_only_two_forward_pages() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
fail_once(&storage, "a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c"]);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
|
||||
execution.await.expect("all objects complete");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c", "a", "d"]);
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() {
|
||||
let mut storage = pages_storage(&[&["a"], &["b", "c"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(3));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
bucket_task(storage.clone()).execute().await.expect("all objects complete");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "c"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
|
||||
let mut storage = pages_storage(&[&["a"], &["b"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("aged pending responsibility is not success");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(outcome.counters.failed, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a outcome")
|
||||
.disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_drains_owned_retries_once() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
|
||||
fail_once(&storage, "a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
task.cancel().await.expect("cancel");
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == "a").count(), 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a")
|
||||
.disposition,
|
||||
HealObjectDisposition::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn deadline_drains_owned_retries_without_false_completion() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
|
||||
fail_once(&storage, "a");
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.options.timeout = Some(Duration::from_secs(1));
|
||||
assert!(matches!(task.execute().await, Err(Error::TaskTimeout)));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a")
|
||||
.disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None
|
||||
}
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn full_window_backpressures_without_losing_the_current_page_tail() {
|
||||
let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
|
||||
let mut storage = MockStorage {
|
||||
retry_test_pages: Some(vec![names.iter().map(|name| heal_item(name)).collect(), vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
};
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
}
|
||||
// The last item has a version, proving the current-page tail is not rebuilt
|
||||
// from names alone when the window fills.
|
||||
storage.retry_test_pages.as_mut().expect("pages")[0][257].version_id = Some("version-tail".to_string());
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
execution.await.expect("every owned item eventually completes");
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 259);
|
||||
let calls = storage.heal_object_calls.lock().expect("calls");
|
||||
for name in &names {
|
||||
assert_eq!(calls.iter().filter(|called| *called == name).count(), 2);
|
||||
}
|
||||
let versions = storage.heal_object_version_ids.lock().expect("versions");
|
||||
for (name, version) in calls.iter().zip(versions.iter()) {
|
||||
if name == "blocked-257" {
|
||||
assert_eq!(version.as_deref(), Some("version-tail"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn oversized_identity_stays_inline_without_losing_version() {
|
||||
let name = "k".repeat(256 * 1024);
|
||||
let mut item = heal_item(&name);
|
||||
item.version_id = Some("v".repeat(1024));
|
||||
let storage = Arc::new(MockStorage {
|
||||
retry_test_pages: Some(vec![vec![item], vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, &name);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
execution.await.expect("oversized identity retries inline");
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 2);
|
||||
let versions = storage.heal_object_version_ids.lock().expect("versions");
|
||||
assert_eq!(versions[0], versions[1]);
|
||||
assert_eq!(versions[0].as_ref().expect("version").len(), 1024);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn terminal_listing_failure_keeps_deferred_identity_unknown() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
fail_second_listing_page: true,
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("listing cannot continue");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
|
||||
assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn healthy_second_page_advances_before_first_retry_is_due() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(0)),
|
||||
..Default::default()
|
||||
});
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableSlowDown]));
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err(),
|
||||
"the deferred first object must remain pending before its retry is due"
|
||||
);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-b"],
|
||||
"a retryable page head must not hold the healthy second page behind its backoff"
|
||||
);
|
||||
execution.await.expect("retry eventually succeeds");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-b", "object-a"]
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn item(name: String, version_id: Option<String>) -> DeferredObject {
|
||||
DeferredObject::new(
|
||||
HealListItem {
|
||||
name,
|
||||
version_id,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
},
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn count_cap_and_next_item_preserve_ownership() {
|
||||
let mut window = DeferredWindow::default();
|
||||
for _ in 0..MAX_DEFERRED_OBJECTS {
|
||||
assert!(window.push(item("key".to_string(), None)).is_ok());
|
||||
}
|
||||
let rejected = window
|
||||
.push(item("next".to_string(), Some("version".to_string())))
|
||||
.expect_err("count cap");
|
||||
assert_eq!(rejected.name, "next");
|
||||
assert_eq!(rejected.version_id.as_deref(), Some("version"));
|
||||
assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS);
|
||||
assert!(window.bytes <= MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.pop_due().is_some());
|
||||
assert!(window.push(rejected).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn byte_cap_counts_key_version_and_reserved_slots() {
|
||||
let mut window = DeferredWindow::default();
|
||||
let available = MAX_DEFERRED_BYTES - window.bytes;
|
||||
let key = "k".repeat(available / 2);
|
||||
let version = "v".repeat(available - key.capacity());
|
||||
assert_eq!(key.capacity() + version.capacity(), available);
|
||||
assert!(window.push(item(key, Some(version))).is_ok());
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.push(item("x".to_string(), None)).is_err());
|
||||
assert!(window.pop_due().is_some());
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>());
|
||||
assert!(window.push(item("x".to_string(), None)).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() {
|
||||
let mut entry = item("a".to_string(), None);
|
||||
entry.defer(Duration::from_secs(2));
|
||||
let first = entry.first_failure.expect("first failure");
|
||||
tokio::time::advance(Duration::from_secs(29)).await;
|
||||
entry.defer(Duration::from_secs(8));
|
||||
assert_eq!(entry.first_failure, Some(first));
|
||||
assert_eq!(entry.due, first + MAX_DEFERRED_AGE);
|
||||
assert!(!entry.expired());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert!(entry.expired());
|
||||
}
|
||||
@@ -66,9 +66,8 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
|
||||
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
|
||||
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
|
||||
const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer";
|
||||
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str =
|
||||
"Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And";
|
||||
const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates";
|
||||
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str = "Filter has too many predicates";
|
||||
const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key";
|
||||
const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters";
|
||||
const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative";
|
||||
|
||||
+209
-5
@@ -167,6 +167,13 @@ pub struct HealTaskStatus {
|
||||
/// Live progress snapshot; the exact shape is owned by the heal runtime.
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Canonical heal-owner result. Missing or future states are not repair proof.
|
||||
#[serde(default)]
|
||||
pub outcome: Option<serde_json::Value>,
|
||||
#[serde(default, alias = "next_seq")]
|
||||
pub next_seq: Option<u64>,
|
||||
#[serde(default, alias = "min_seq")]
|
||||
pub min_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// `POST /v3/background-heal/status` response. Known top-level fields are
|
||||
@@ -223,6 +230,40 @@ pub struct ScannerStatus {
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/scanner/cycle-state/reset` response for the legacy synchronous
|
||||
/// full-rescan reset path.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ScannerCycleResetResponse {
|
||||
pub status: String,
|
||||
pub mode: String,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/scanner/usage-state/reset` response for the legacy synchronous
|
||||
/// full-rebuild reset path.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ScannerUsageStateResetResponse {
|
||||
pub status: String,
|
||||
pub mode: String,
|
||||
pub usage_state: String,
|
||||
pub leader_epoch: u64,
|
||||
pub next_cycle: u64,
|
||||
pub reset_paths: Vec<String>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ScannerCycleResetRequest<'a> {
|
||||
mode: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ScannerUsageStateResetRequest<'a> {
|
||||
mode: &'a str,
|
||||
}
|
||||
|
||||
/// Freshness block of the scanner status response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -358,8 +399,22 @@ impl AdminClient {
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
|
||||
.await
|
||||
self.heal_status_since(bucket, prefix, client_token, None).await
|
||||
}
|
||||
|
||||
/// Query a retained result window. Missing cursors and outcome remain unknown.
|
||||
pub async fn heal_status_since(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
let mut query = vec![("clientToken", client_token.to_string())];
|
||||
if let Some(since_seq) = since_seq {
|
||||
query.push(("sinceSeq", since_seq.to_string()));
|
||||
}
|
||||
self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await
|
||||
}
|
||||
|
||||
/// Stop a heal: with a `client_token` only that task is cancelled and its
|
||||
@@ -378,7 +433,7 @@ impl AdminClient {
|
||||
match client_token {
|
||||
Some(_) => {
|
||||
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::Stopped(status))
|
||||
Ok(HealStopOutcome::Stopped(Box::new(status)))
|
||||
}
|
||||
None => {
|
||||
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
@@ -398,6 +453,25 @@ impl AdminClient {
|
||||
self.get_json("/v3/scanner/status").await
|
||||
}
|
||||
|
||||
/// Request a legacy synchronous scanner cycle reset (`full-rescan`).
|
||||
pub async fn scanner_cycle_state_reset_full_rescan(&self) -> Result<ScannerCycleResetResponse, AdminClientError> {
|
||||
let body =
|
||||
serde_json::to_vec(&ScannerCycleResetRequest { mode: "full-rescan" }).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
self.post_json("/v3/scanner/cycle-state/reset", &[], body).await
|
||||
}
|
||||
|
||||
/// Request a legacy synchronous scanner usage reset (`full-rebuild`).
|
||||
pub async fn scanner_usage_state_reset_full_rebuild(&self) -> Result<ScannerUsageStateResetResponse, AdminClientError> {
|
||||
let body = serde_json::to_vec(&ScannerUsageStateResetRequest { mode: "full-rebuild" }).map_err(|err| {
|
||||
AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
}
|
||||
})?;
|
||||
self.post_json("/v3/scanner/usage-state/reset", &[], body).await
|
||||
}
|
||||
|
||||
/// ILM expiry worker status. The payload is owned by the expiry
|
||||
/// subsystem and still evolving; returned verbatim.
|
||||
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
@@ -533,7 +607,7 @@ impl AdminClient {
|
||||
/// start-success-shaped receipt.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealStopOutcome {
|
||||
Stopped(HealTaskStatus),
|
||||
Stopped(Box<HealTaskStatus>),
|
||||
PathStopped(HealStartSuccess),
|
||||
}
|
||||
|
||||
@@ -566,7 +640,7 @@ pub(crate) fn percent_encode_path_segment(segment: &str) -> String {
|
||||
mod tests {
|
||||
use super::{
|
||||
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
|
||||
ScannerStatus, heal_path, percent_encode_path_segment,
|
||||
ScannerCycleResetResponse, ScannerStatus, ScannerUsageStateResetResponse, heal_path, percent_encode_path_segment,
|
||||
};
|
||||
use crate::test_support::TestServer;
|
||||
use serde_json::json;
|
||||
@@ -635,6 +709,24 @@ mod tests {
|
||||
assert!(status.progress.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_decoder_preserves_canonical_unknown_and_future_fields() {
|
||||
let cases: serde_json::Value =
|
||||
serde_json::from_str(include_str!("../tests/fixtures/heal-outcome-v3.json")).expect("shared fixtures");
|
||||
for case in cases.as_array().expect("cases") {
|
||||
let status: HealTaskStatus = serde_json::from_value(case["response"].clone()).expect("optional outcome response");
|
||||
assert_eq!(status.outcome.as_ref(), Some(&case["response"]["outcome"]));
|
||||
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
|
||||
assert!(status.truncated);
|
||||
}
|
||||
let old: HealTaskStatus = serde_json::from_value(json!({"summary":"finished"})).expect("legacy response");
|
||||
assert!(old.outcome.is_none() && old.next_seq.is_none() && old.min_seq.is_none());
|
||||
let future = json!({"execution":{"state":"future_state"},"newField":7});
|
||||
let status: HealTaskStatus =
|
||||
serde_json::from_value(json!({"summary":"running","outcome":future})).expect("future outcome remains opaque");
|
||||
assert_eq!(status.outcome, Some(future));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
|
||||
let raw = json!({
|
||||
@@ -691,6 +783,33 @@ mod tests {
|
||||
assert_eq!(bare.freshness(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_reset_responses_preserve_future_fields() {
|
||||
let cycle: ScannerCycleResetResponse =
|
||||
serde_json::from_value(json!({"status": "reset", "mode": "full-rescan", "future": true})).unwrap();
|
||||
assert_eq!(cycle.status, "reset");
|
||||
assert_eq!(cycle.mode, "full-rescan");
|
||||
assert_eq!(cycle.extra["future"], true);
|
||||
|
||||
let usage: ScannerUsageStateResetResponse = serde_json::from_value(json!({
|
||||
"status": "reset",
|
||||
"mode": "full-rebuild",
|
||||
"usage_state": "bootstrap-pending",
|
||||
"leader_epoch": 11,
|
||||
"next_cycle": 42,
|
||||
"reset_paths": [".usage.json"],
|
||||
"future": {"accepted": false}
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(usage.status, "reset");
|
||||
assert_eq!(usage.mode, "full-rebuild");
|
||||
assert_eq!(usage.usage_state, "bootstrap-pending");
|
||||
assert_eq!(usage.leader_epoch, 11);
|
||||
assert_eq!(usage.next_cycle, 42);
|
||||
assert_eq!(usage.reset_paths, [".usage.json"]);
|
||||
assert_eq!(usage.extra["future"]["accepted"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_endpoint_is_rejected_without_io() {
|
||||
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
|
||||
@@ -733,6 +852,52 @@ mod tests {
|
||||
assert!(request.body.contains("\"recursive\":true"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_reset_posts_legacy_full_rescan_request() {
|
||||
let server = TestServer::spawn(r#"{"status":"reset","mode":"full-rescan"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let reset = client
|
||||
.scanner_cycle_state_reset_full_rescan()
|
||||
.await
|
||||
.expect("cycle reset response decodes");
|
||||
|
||||
assert_eq!(reset.status, "reset");
|
||||
assert_eq!(reset.mode, "full-rescan");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/cycle-state/reset");
|
||||
assert_eq!(request.query, "");
|
||||
assert!(request.body.contains("\"mode\":\"full-rescan\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_reset_posts_legacy_full_rebuild_request() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"reset","mode":"full-rebuild","usage_state":"bootstrap-pending","leader_epoch":11,"next_cycle":42,"reset_paths":[".usage.json"]}"#,
|
||||
200,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let reset = client
|
||||
.scanner_usage_state_reset_full_rebuild()
|
||||
.await
|
||||
.expect("usage reset response decodes");
|
||||
|
||||
assert_eq!(reset.status, "reset");
|
||||
assert_eq!(reset.mode, "full-rebuild");
|
||||
assert_eq!(reset.usage_state, "bootstrap-pending");
|
||||
assert_eq!(reset.leader_epoch, 11);
|
||||
assert_eq!(reset.next_cycle, 42);
|
||||
assert_eq!(reset.reset_paths, [".usage.json"]);
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||
assert_eq!(request.query, "");
|
||||
assert!(request.body.contains("\"mode\":\"full-rebuild\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_sends_client_token_on_the_same_path() {
|
||||
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
@@ -750,6 +915,26 @@ mod tests {
|
||||
assert!(!request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outcome_v3_since_query_preserves_cursor_and_never_sends_force_start() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"summary":"running","nextSeq":9,"minSeq":4,"truncated":true,"outcome":{"execution":{"state":"future_state"}}}"#,
|
||||
200,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("test client");
|
||||
let status = client
|
||||
.heal_status_since(Some("bucket"), None, "token-1", Some(3))
|
||||
.await
|
||||
.expect("window response");
|
||||
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
|
||||
assert!(status.truncated);
|
||||
assert_eq!(status.outcome.expect("future state is preserved")["execution"]["state"], "future_state");
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("sinceSeq=3") && request.query.contains("clientToken=token-1"));
|
||||
assert!(!request.query.contains("forceStart") && !request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_without_token_takes_the_path_cancel_branch() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
|
||||
@@ -762,6 +947,25 @@ mod tests {
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_with_token_decodes_boxed_task_status() {
|
||||
let body = r#"{"summary":"stopped","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let outcome = client
|
||||
.heal_stop(Some("bucket"), None, Some("token-1"))
|
||||
.await
|
||||
.expect("token stop decodes");
|
||||
let super::HealStopOutcome::Stopped(status) = outcome else {
|
||||
panic!("token stop should return task status");
|
||||
};
|
||||
assert_eq!(status.summary, "stopped");
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("forceStop=true"));
|
||||
assert!(request.query.contains("clientToken=token-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_heal_status_posts_to_the_registered_route() {
|
||||
let body = r#"{"state":"idle","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":true}"#;
|
||||
|
||||
+472
@@ -0,0 +1,472 @@
|
||||
[
|
||||
{
|
||||
"name": "completed",
|
||||
"cliExit": 0,
|
||||
"response": {
|
||||
"summary": "finished",
|
||||
"detail": "heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "unknown",
|
||||
"cliExit": 0,
|
||||
"response": {
|
||||
"summary": "finished",
|
||||
"detail": "heal traversal completed; authoritative storage proof is unavailable for 1 objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 1,
|
||||
"failed": 0,
|
||||
"unknown": 1,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "unknown"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "completed_with_errors",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "cancelled",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task cancelled; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "deadline",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task timed out; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "deadline"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "untraversable",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal listing is untraversable; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "untraversable"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "remote_completed_with_errors",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
},
|
||||
"remoteResponse": {
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "remote_cancelled",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task cancelled; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
},
|
||||
"remoteResponse": {
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1257,6 +1257,8 @@ pub struct ScannerDirtyUsageBucket {
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub generation: u64,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub bucket_incarnation: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -1282,6 +1284,8 @@ pub struct ScannerDirtyUsageSnapshotResponse {
|
||||
pub buckets: ::prost::alloc::vec::Vec<ScannerDirtyUsageBucket>,
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "8")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
}
|
||||
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
/// have a durable per-bucket publication proof.
|
||||
|
||||
@@ -175,6 +175,9 @@ pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0";
|
||||
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
|
||||
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
|
||||
pub const ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-ilm-recovery-export-capability-v1\0";
|
||||
pub const TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX: &[u8] =
|
||||
b"rustfs-transition-transaction-compaction-capability-v1\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
@@ -219,6 +222,30 @@ pub fn is_cross_pool_fence_capability_probe(command: &[u8]) -> bool {
|
||||
&& command.starts_with(CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_ilm_recovery_export_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn transition_transaction_compaction_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_transition_transaction_compaction_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn encode_remote_version_state_capability(
|
||||
topology_member: &str,
|
||||
process_epoch: &[u8; 16],
|
||||
@@ -562,11 +589,13 @@ pub fn canonical_scanner_dirty_usage_snapshot_response_body(
|
||||
body.push_u64(response.generation);
|
||||
body.push_u64(response.pending_bucket_count);
|
||||
body.push_u32(response.protocol_version);
|
||||
body.push_str(&response.owner_id)?;
|
||||
body.push_bool(response.complete);
|
||||
body.push_count(response.buckets.len())?;
|
||||
for bucket in &response.buckets {
|
||||
body.push_str(&bucket.bucket)?;
|
||||
body.push_u64(bucket.generation);
|
||||
body.push_bytes(bucket.bucket_incarnation.as_ref())?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
@@ -1829,13 +1858,16 @@ mod scanner_activity_tests {
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
},
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: vec![2; 16].into(),
|
||||
},
|
||||
],
|
||||
response_proof: vec![9; 32].into(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
};
|
||||
let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response)
|
||||
.expect("scanner dirty usage snapshot response should encode");
|
||||
@@ -1852,6 +1884,9 @@ mod scanner_activity_tests {
|
||||
let mut protocol = response.clone();
|
||||
protocol.protocol_version = 2;
|
||||
variants.push(protocol);
|
||||
let mut owner = response.clone();
|
||||
owner.owner_id = "22222222-2222-2222-2222-222222222222".to_string();
|
||||
variants.push(owner);
|
||||
let mut complete = response.clone();
|
||||
complete.complete = false;
|
||||
variants.push(complete);
|
||||
@@ -1861,6 +1896,9 @@ mod scanner_activity_tests {
|
||||
let mut bucket_generation = response.clone();
|
||||
bucket_generation.buckets[0].generation = 4;
|
||||
variants.push(bucket_generation);
|
||||
let mut bucket_incarnation = response.clone();
|
||||
bucket_incarnation.buckets[0].bucket_incarnation = vec![3; 16].into();
|
||||
variants.push(bucket_incarnation);
|
||||
let mut bucket_order = response.clone();
|
||||
bucket_order.buckets.reverse();
|
||||
variants.push(bucket_order);
|
||||
@@ -2127,12 +2165,15 @@ mod scanner_activity_tests {
|
||||
mod heal_control_tests {
|
||||
use super::{
|
||||
CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack, canonical_heal_control_request_body,
|
||||
canonical_heal_control_response_body, decode_remote_version_state_capability, encode_cross_pool_fence_capability,
|
||||
encode_remote_version_state_capability, heal_control_capability_probe, heal_control_coordinator_epoch,
|
||||
heal_control_execution_timeout, heal_control_execution_timeout_for, internode_rpc_timeout,
|
||||
is_cross_pool_fence_capability_probe, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
|
||||
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
|
||||
ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
|
||||
TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack,
|
||||
canonical_heal_control_request_body, canonical_heal_control_response_body, decode_remote_version_state_capability,
|
||||
encode_cross_pool_fence_capability, encode_remote_version_state_capability, heal_control_capability_probe,
|
||||
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
|
||||
ilm_recovery_export_capability_probe, internode_rpc_timeout, is_cross_pool_fence_capability_probe,
|
||||
is_heal_control_capability_probe, is_ilm_recovery_export_capability_probe, is_remote_version_state_capability_probe,
|
||||
is_transition_transaction_compaction_capability_probe, normalize_internode_rpc_timeout,
|
||||
remote_version_state_capability_probe, transition_transaction_compaction_capability_probe,
|
||||
};
|
||||
use crate::heal_control;
|
||||
use std::time::Duration;
|
||||
@@ -2196,6 +2237,34 @@ mod heal_control_tests {
|
||||
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_capability_probe_requires_exact_prefix_and_nonce() {
|
||||
let probe = ilm_recovery_export_capability_probe(&[7; 16]);
|
||||
assert!(is_ilm_recovery_export_capability_probe(&probe));
|
||||
assert!(!is_ilm_recovery_export_capability_probe(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX));
|
||||
let mut wrong_prefix = probe.clone();
|
||||
wrong_prefix[0] ^= 1;
|
||||
assert!(!is_ilm_recovery_export_capability_probe(&wrong_prefix));
|
||||
let mut extra = probe;
|
||||
extra.push(0);
|
||||
assert!(!is_ilm_recovery_export_capability_probe(&extra));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_transaction_compaction_probe_requires_exact_prefix_and_nonce() {
|
||||
let probe = transition_transaction_compaction_capability_probe(&[7; 16]);
|
||||
assert!(is_transition_transaction_compaction_capability_probe(&probe));
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(
|
||||
TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX,
|
||||
));
|
||||
let mut wrong_prefix = probe.clone();
|
||||
wrong_prefix[0] ^= 1;
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(&wrong_prefix));
|
||||
let mut extra = probe;
|
||||
extra.push(0);
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(&extra));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_binds_member_and_process_epoch() {
|
||||
let encoded =
|
||||
|
||||
@@ -885,6 +885,7 @@ message ScannerActivityResponse {
|
||||
message ScannerDirtyUsageBucket {
|
||||
string bucket = 1;
|
||||
uint64 generation = 2;
|
||||
bytes bucket_incarnation = 3;
|
||||
}
|
||||
|
||||
message ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -901,6 +902,7 @@ message ScannerDirtyUsageSnapshotResponse {
|
||||
bool complete = 5;
|
||||
repeated ScannerDirtyUsageBucket buckets = 6;
|
||||
bytes response_proof = 7;
|
||||
string owner_id = 8;
|
||||
}
|
||||
|
||||
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
|
||||
@@ -65,9 +65,10 @@ pub use multipart::{
|
||||
};
|
||||
pub use object::{
|
||||
ObjectLockIntegrity, ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
|
||||
content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity, replication_action_for_target,
|
||||
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity,
|
||||
replication_action_for_target, replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present,
|
||||
ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
|
||||
@@ -234,18 +234,62 @@ fn comparable_metadata(metadata: Option<&HashMap<String, String>>) -> HashMap<St
|
||||
/// real (non-nil) version uuid, and drift means the target answered with
|
||||
/// anything else — including nothing at all.
|
||||
pub fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
|
||||
if source_version_id.is_empty() {
|
||||
return false;
|
||||
version_identity_capability_from_put(source_version_id, assigned_version_id) == Some(VersionIdentityCapability::MintsOwn)
|
||||
}
|
||||
|
||||
/// Whether a replication target adopts the source version id it is handed on
|
||||
/// PutObject / CompleteMultipartUpload, or mints its own.
|
||||
///
|
||||
/// A target that mints its own ids (AWS S3, Wasabi, Impossible Cloud) still
|
||||
/// stores the bytes, but every later version-addressed request from the
|
||||
/// source names an id the target never had. Its HEAD then answers 404 —
|
||||
/// indistinguishable from a replica that is really missing — so a heal, MRF
|
||||
/// retry or existing-object resync re-drive would PUT the object again and
|
||||
/// mint yet another target version (rustfs/backlog#2340). The replication
|
||||
/// worker learns the verdict from each PUT response (and replication-check's
|
||||
/// VersionFidelity phase) and, once `MintsOwn` is known, locates a replica by
|
||||
/// exact key and ETag before concluding that it is missing. The verdict cache
|
||||
/// is owned by the runtime's bucket target system; this crate owns only the
|
||||
/// vocabulary and the judgment.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum VersionIdentityCapability {
|
||||
#[default]
|
||||
Unknown,
|
||||
Adopts,
|
||||
MintsOwn,
|
||||
}
|
||||
|
||||
impl VersionIdentityCapability {
|
||||
/// True when a 404 from a version-addressed HEAD on this target cannot be
|
||||
/// read as "replica missing": the source-side id was never the target's.
|
||||
pub fn version_addressing_unreliable(self) -> bool {
|
||||
self == VersionIdentityCapability::MintsOwn
|
||||
}
|
||||
}
|
||||
|
||||
/// Judge the identity contract from one replication write: `None` when no
|
||||
/// contract applies (the source addressed no real version — an empty or nil
|
||||
/// uuid travels as the literal "null", unversioned-source semantics),
|
||||
/// otherwise whether the target echoed the source id or answered with
|
||||
/// anything else — including nothing at all.
|
||||
pub fn version_identity_capability_from_put(
|
||||
source_version_id: &str,
|
||||
assigned_version_id: Option<&str>,
|
||||
) -> Option<VersionIdentityCapability> {
|
||||
if source_version_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// A nil source uuid travels as the literal "null" (unversioned-source
|
||||
// semantics); no identity contract applies to it.
|
||||
if uuid::Uuid::parse_str(source_version_id)
|
||||
.map(|uuid| uuid.is_nil())
|
||||
.unwrap_or(true)
|
||||
{
|
||||
return false;
|
||||
return None;
|
||||
}
|
||||
assigned_version_id != Some(source_version_id)
|
||||
Some(if assigned_version_id == Some(source_version_id) {
|
||||
VersionIdentityCapability::Adopts
|
||||
} else {
|
||||
VersionIdentityCapability::MintsOwn
|
||||
})
|
||||
}
|
||||
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
@@ -377,9 +421,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
|
||||
content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
|
||||
single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target,
|
||||
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_capability_from_put, version_identity_drifted,
|
||||
};
|
||||
use crate::filemeta::{ReplicationAction, ReplicationType};
|
||||
use crate::http::AMZ_OBJECT_LOCK_MODE;
|
||||
@@ -508,6 +552,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_capability_is_judged_only_for_real_source_versions() {
|
||||
let source = "8e4d2f4c-2d5c-4f1b-9d0a-9c8b7a6f5e4d";
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, Some(source)),
|
||||
Some(VersionIdentityCapability::Adopts)
|
||||
);
|
||||
// Wasabi / AWS shape: a minted id, or no id at all, both mean the
|
||||
// source-side id is not addressable on the target.
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, Some("001788697733811332140-fR6j6uXKV-")),
|
||||
Some(VersionIdentityCapability::MintsOwn)
|
||||
);
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, None),
|
||||
Some(VersionIdentityCapability::MintsOwn)
|
||||
);
|
||||
// No contract for an unversioned source write.
|
||||
assert_eq!(version_identity_capability_from_put("", Some("anything")), None);
|
||||
assert_eq!(version_identity_capability_from_put("00000000-0000-0000-0000-000000000000", None), None);
|
||||
assert_eq!(version_identity_capability_from_put("null", Some("null")), None);
|
||||
assert!(VersionIdentityCapability::MintsOwn.version_addressing_unreliable());
|
||||
assert!(!VersionIdentityCapability::Adopts.version_addressing_unreliable());
|
||||
assert!(!VersionIdentityCapability::Unknown.version_addressing_unreliable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_offline_error_classifier_is_network_scoped() {
|
||||
assert!(is_replication_target_offline_error("put_object dispatch failure: connector error"));
|
||||
|
||||
@@ -106,6 +106,7 @@ bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-heal.workspace = true
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -85,14 +85,14 @@ pub use rustfs_scanner_metrics::last_minute;
|
||||
pub use scanner::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
|
||||
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
|
||||
init_data_scanner, reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
|
||||
scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
|
||||
init_data_scanner, init_scanner_with_recovery, reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild,
|
||||
scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
@@ -100,6 +100,7 @@ pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
|
||||
|
||||
|
||||
@@ -16,8 +16,9 @@
|
||||
use crate::RUSTFS_META_BUCKET;
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
|
||||
use crate::scanner_io::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks,
|
||||
cache_root_entry_info, current_cache_root_or_prepare_with_generation, scanner_set_disk_inventory,
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk,
|
||||
acquire_scanner_cache_locks, cache_root_entry_info, current_cache_root_or_prepare_with_generation,
|
||||
scanner_set_disk_inventory,
|
||||
};
|
||||
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::{
|
||||
@@ -779,7 +780,18 @@ async fn scan_and_persist_local_bucket(
|
||||
|
||||
let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
|
||||
let scan_ctx = ctx.child_token();
|
||||
let scan = ScannerIODisk::nsscanner_disk(disk.clone(), scan_ctx.clone(), budget, set_disks, cache, None, scan_mode);
|
||||
let scan = ScannerIODisk::nsscanner_disk(
|
||||
disk.clone(),
|
||||
scan_ctx.clone(),
|
||||
budget,
|
||||
set_disks,
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
},
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL);
|
||||
tokio::pin!(fence_watch);
|
||||
|
||||
+122
-28
@@ -14,7 +14,6 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
@@ -948,6 +947,33 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
init_data_scanner_with_storage(ctx, storeapi).await;
|
||||
}
|
||||
|
||||
/// Start normal scanning when enabled, or one resume-only cleanup attempt.
|
||||
/// The disabled branch returns a finite task for the startup owner to join;
|
||||
/// it never enables ordinary namespace scanning or accepts a new reset intent.
|
||||
pub async fn init_scanner_with_recovery(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
enabled: bool,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
if enabled {
|
||||
init_data_scanner(ctx, storeapi).await;
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(error) = resume_scanner_cycle_cleanup(ctx, storeapi).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "disabled_cleanup_deferred",
|
||||
error = %error,
|
||||
"Disabled scanner cleanup remains pending for an operator retry"
|
||||
);
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn init_data_scanner_with_storage<S>(ctx: CancellationToken, storeapi: Arc<S>)
|
||||
where
|
||||
S: ScannerStorage,
|
||||
@@ -1558,6 +1584,11 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
}
|
||||
|
||||
struct ScannerCycleScheduling {
|
||||
requires_full_scan: bool,
|
||||
service_cohort: Option<Arc<StdMutex<crate::scanner_io::ScannerServiceCohort>>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn run_data_scanner_cycle<S>(
|
||||
ctx: &CancellationToken,
|
||||
@@ -1570,7 +1601,19 @@ where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
|
||||
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget, true).await
|
||||
run_data_scanner_cycle_with_budget(
|
||||
ctx,
|
||||
storeapi,
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget,
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
@@ -1582,7 +1625,7 @@ async fn run_data_scanner_cycle_with_budget<S>(
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_budget: Arc<ScannerCycleBudget>,
|
||||
requires_full_scan: bool,
|
||||
scheduling: ScannerCycleScheduling,
|
||||
) -> ScannerCycleOutcome
|
||||
where
|
||||
S: ScannerStorage,
|
||||
@@ -1687,6 +1730,7 @@ where
|
||||
};
|
||||
let baseline_publication_epoch = baseline_publication_guard.epoch();
|
||||
let usage_persist_baseline_result = read_data_usage_persist_baseline(storeapi.clone()).await;
|
||||
let observed_usage_candidate_result = read_config(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
|
||||
drop(baseline_publication_guard);
|
||||
let usage_persist_baseline = match usage_persist_baseline_result {
|
||||
Ok(baseline) => baseline,
|
||||
@@ -1707,6 +1751,24 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
};
|
||||
let observed_usage_candidate = match observed_usage_candidate_result {
|
||||
Ok(candidate) => Some(Bytes::from(candidate)),
|
||||
Err(EcstoreError::ConfigNotFound) => None,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_candidate_load_failed",
|
||||
error = %err,
|
||||
"Scanner skipped an unavailable observed usage candidate for scoped refresh"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
@@ -1721,7 +1783,9 @@ where
|
||||
scan_mode,
|
||||
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: usage_persist_baseline.data.clone(),
|
||||
requires_full_scan,
|
||||
observed_usage_candidate,
|
||||
requires_full_scan: scheduling.requires_full_scan,
|
||||
service_cohort: scheduling.service_cohort,
|
||||
#[cfg(test)]
|
||||
resolved_scope_observer: None,
|
||||
},
|
||||
@@ -1846,10 +1910,10 @@ where
|
||||
.as_ref()
|
||||
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
|
||||
let remote_lease_release_safe = Arc::new(AtomicBool::new(true));
|
||||
let mut usage_persist_outcome = match publication_defer_reason {
|
||||
let mut usage_publication_result = match publication_defer_reason {
|
||||
Some(reason) => {
|
||||
drop(receiver);
|
||||
DataUsagePersistOutcome::Deferred(reason)
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Deferred(reason))
|
||||
}
|
||||
None => {
|
||||
// ScannerIO emits its complete or observational update only after
|
||||
@@ -1864,6 +1928,11 @@ where
|
||||
.as_ref()
|
||||
.map(|(_, grants)| grants.iter().map(|grant| grant.lease.token).collect())
|
||||
.unwrap_or_default();
|
||||
let ack_expectation = scan_result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.filter(|result| result.has_dirty_usage_to_acknowledge())
|
||||
.and_then(ScannerCycleResult::publication_expectation);
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
ctx_clone,
|
||||
@@ -1876,6 +1945,7 @@ where
|
||||
remote_lease_deadline,
|
||||
remote_lease_fence,
|
||||
)
|
||||
.with_ack_expectation(ack_expectation)
|
||||
.with_remote_lease_tokens(remote_lease_tokens)
|
||||
.with_lease_release_flag(remote_lease_release_safe_for_task),
|
||||
move || {
|
||||
@@ -1909,7 +1979,7 @@ where
|
||||
error = %err,
|
||||
"Scanner data usage persistence task failed"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
}
|
||||
DataUsagePersistTaskResult::Cancelled => {
|
||||
debug!(
|
||||
@@ -1921,7 +1991,7 @@ where
|
||||
state = "usage_persist_task_cancelled",
|
||||
"Scanner data usage persistence task cancelled"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
}
|
||||
DataUsagePersistTaskResult::TimedOut => {
|
||||
error!(
|
||||
@@ -1934,11 +2004,12 @@ where
|
||||
state = "usage_persist_task_timed_out",
|
||||
"Scanner data usage persistence task timed out"
|
||||
);
|
||||
DataUsagePersistOutcome::Failed
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut usage_persist_outcome = usage_publication_result.outcome();
|
||||
let lease_expired = remote_publication_leases
|
||||
.as_ref()
|
||||
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
|
||||
@@ -2138,16 +2209,14 @@ where
|
||||
};
|
||||
}
|
||||
|
||||
usage_publication_result.restrict_outcome(usage_persist_outcome);
|
||||
let (completion_outcome, scanner_pending_maintenance_work, remote_dirty_usage_acknowledgements) =
|
||||
finalize_scanner_cycle_result(scan_cycle_result, usage_persist_outcome);
|
||||
finalize_scanner_cycle_result(scan_cycle_result, usage_publication_result);
|
||||
let remote_dirty_usage_pending = if remote_dirty_usage_acknowledgements.is_empty() {
|
||||
false
|
||||
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
|
||||
let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements
|
||||
.into_iter()
|
||||
.map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation))
|
||||
.collect();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements.into_iter().map(Into::into).collect();
|
||||
remote_dirty_usage_acknowledgement_pending(
|
||||
cycle_info.current,
|
||||
acknowledgement_count,
|
||||
@@ -2583,6 +2652,7 @@ where
|
||||
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
|
||||
let mut superseded_backoff = ScannerRetryBackoff::default();
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
||||
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
||||
@@ -2792,7 +2862,10 @@ where
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
true,
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: Some(service_cohort.clone()),
|
||||
},
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
@@ -3083,11 +3156,14 @@ where
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
maintenance_features.requires_full_scan(
|
||||
maintenance_generation_seen,
|
||||
scanner_maintenance_generation(),
|
||||
wake_reason,
|
||||
),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: maintenance_features.requires_full_scan(
|
||||
maintenance_generation_seen,
|
||||
scanner_maintenance_generation(),
|
||||
wake_reason,
|
||||
),
|
||||
service_cohort: Some(service_cohort.clone()),
|
||||
},
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
@@ -3366,21 +3442,35 @@ fn scanner_cycle_completion_outcome(
|
||||
|
||||
fn finalize_scanner_cycle_result(
|
||||
scan_cycle_result: crate::scanner_io::ScannerCycleResult,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
publication: DataUsagePublicationResult,
|
||||
) -> (ScannerCycleOutcome, bool, Vec<ScannerDirtyUsageAcknowledgement>) {
|
||||
let (usage_persist_outcome, proof) = publication.into_parts();
|
||||
let completion_outcome = scanner_cycle_completion_outcome_for_result(&scan_cycle_result, usage_persist_outcome);
|
||||
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work();
|
||||
let durable_complete_snapshot = scan_cycle_result.status == ScannerCycleStatus::Complete
|
||||
&& matches!(
|
||||
usage_persist_outcome,
|
||||
DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable
|
||||
);
|
||||
)
|
||||
&& scan_cycle_result.publication_expectation().as_ref().is_some_and(|expected| {
|
||||
proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| proof.verified_version_for(expected).is_some())
|
||||
});
|
||||
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work()
|
||||
|| (scan_cycle_result.has_dirty_usage_to_acknowledge() && !durable_complete_snapshot);
|
||||
let remote_dirty_usage_acknowledgements = if durable_complete_snapshot {
|
||||
scan_cycle_result.acknowledge_durable_usage()
|
||||
match proof {
|
||||
Some(proof) => scan_cycle_result.acknowledge_durable_usage(&proof),
|
||||
None => Vec::new(),
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
(completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements)
|
||||
(
|
||||
completion_outcome,
|
||||
pending_maintenance_work || crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
remote_dirty_usage_acknowledgements,
|
||||
)
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome_for_result(
|
||||
@@ -3480,13 +3570,17 @@ use activity::*;
|
||||
use backlog::*;
|
||||
use cycle_state::*;
|
||||
use leadership::*;
|
||||
pub(crate) use usage_store::RootPublicationProof;
|
||||
use usage_store::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use activity::scanner_node_activity_for_tests;
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
||||
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity,
|
||||
scanner_activity_allows_usage_publication, scanner_activity_dirty_usage_state_for_host,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
|
||||
scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -441,11 +441,59 @@ pub(crate) struct ScannerNodeActivity {
|
||||
|
||||
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn scanner_node_activity_for_tests(
|
||||
instance_id: &str,
|
||||
namespace_generation: u64,
|
||||
dirty_usage_generation: u64,
|
||||
dirty_usage_pending: bool,
|
||||
) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: 0,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: [0; 32],
|
||||
data_movement_active: false,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation: 0,
|
||||
publication_blocked: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerDirtyUsageAcknowledgement {
|
||||
pub(crate) host: String,
|
||||
pub(crate) instance_id: String,
|
||||
pub(crate) generation: u64,
|
||||
pub(crate) kind: ScannerDirtyUsageAcknowledgementKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
|
||||
Generation(u64),
|
||||
Scoped {
|
||||
owner_id: String,
|
||||
entries: Vec<crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
|
||||
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
|
||||
match acknowledgement.kind {
|
||||
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Self::Generation {
|
||||
host: acknowledgement.host,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
generation,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } => Self::Scoped {
|
||||
host: acknowledgement.host,
|
||||
owner_id,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
entries,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -974,7 +1022,7 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
|
||||
.map(|(host, activity)| ScannerDirtyUsageAcknowledgement {
|
||||
host: host.clone(),
|
||||
instance_id: activity.instance_id.clone(),
|
||||
generation: activity.dirty_usage_generation,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(activity.dirty_usage_generation),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::data_usage_define::{
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup,
|
||||
};
|
||||
use crate::storage_api::owner::ObjectIO as _;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
const SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION: u16 = 1;
|
||||
@@ -34,6 +35,86 @@ const CACHE_CYCLE_AHEAD: &str = "cache_cycle_ahead";
|
||||
|
||||
const SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD: &str = "full-rebuild";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) mod cleanup_io_fault {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(in crate::scanner) enum Stage {
|
||||
PrimaryRead,
|
||||
PrimaryWrite,
|
||||
UsageFence,
|
||||
}
|
||||
|
||||
struct Injection {
|
||||
store: std::sync::Weak<ECStore>,
|
||||
stage: Stage,
|
||||
fired: AtomicBool,
|
||||
owned: AtomicBool,
|
||||
newer_completion: bool,
|
||||
}
|
||||
|
||||
static INJECTION: StdMutex<Option<Arc<Injection>>> = StdMutex::new(None);
|
||||
pub(in crate::scanner) struct Guard(Arc<Injection>);
|
||||
|
||||
impl Guard {
|
||||
pub(in crate::scanner) fn fired_while_owned(&self) -> bool {
|
||||
self.0.fired.load(Ordering::Relaxed) && self.0.owned.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = INJECTION.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.0)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner) fn install(store: &Arc<ECStore>, stage: Stage, newer_completion: bool) -> Guard {
|
||||
let injection = Arc::new(Injection {
|
||||
store: Arc::downgrade(store),
|
||||
stage,
|
||||
fired: AtomicBool::new(false),
|
||||
owned: AtomicBool::new(false),
|
||||
newer_completion,
|
||||
});
|
||||
let mut slot = INJECTION.lock().expect("cleanup injection slot");
|
||||
assert!(slot.is_none(), "only one cleanup I/O injection may be installed");
|
||||
*slot = Some(injection.clone());
|
||||
Guard(injection)
|
||||
}
|
||||
|
||||
pub(super) fn check(store: &Arc<ECStore>, stage: Stage, owned: bool) -> Result<(), ScannerError> {
|
||||
let injection = {
|
||||
let mut slot = INJECTION.lock().expect("cleanup injection slot");
|
||||
if slot
|
||||
.as_ref()
|
||||
.is_some_and(|injection| injection.stage == stage && injection.store.ptr_eq(&Arc::downgrade(store)))
|
||||
{
|
||||
slot.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let Some(injection) = injection else {
|
||||
return Ok(());
|
||||
};
|
||||
injection.fired.store(true, Ordering::Relaxed);
|
||||
injection.owned.store(owned, Ordering::Relaxed);
|
||||
if injection.newer_completion {
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
}
|
||||
let reason = match stage {
|
||||
Stage::PrimaryRead => "injected primary read failure",
|
||||
Stage::PrimaryWrite => "injected primary write failure",
|
||||
Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
Err(ScannerError::Io(std::io::Error::other(reason)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct ScannerCycleRecoveryStatus {
|
||||
/// The immutable primary object whose revision is being guarded.
|
||||
@@ -81,6 +162,8 @@ static SCANNER_CYCLE_RECOVERY_STATUS: LazyLock<RwLock<ScannerCycleRecoveryStatus
|
||||
..Default::default()
|
||||
})
|
||||
});
|
||||
// An old startup observation must not overwrite a newer explicit reset status.
|
||||
static SCANNER_CYCLE_RECOVERY_STATUS_VERSION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus {
|
||||
SCANNER_CYCLE_RECOVERY_STATUS
|
||||
@@ -90,6 +173,24 @@ pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus {
|
||||
}
|
||||
|
||||
fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) {
|
||||
let _ = publish_scanner_cleanup_status(status, None);
|
||||
}
|
||||
|
||||
pub(super) fn scanner_cleanup_status_version() -> u64 {
|
||||
let _status = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(super) fn publish_scanner_cleanup_status(status: ScannerCycleRecoveryStatus, expected: Option<u64>) -> Option<u64> {
|
||||
let mut current = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let version = SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed);
|
||||
if expected.is_some_and(|expected| expected == u64::MAX || expected != version) {
|
||||
return None;
|
||||
}
|
||||
let recovery_required = if matches!(
|
||||
status.state.as_str(),
|
||||
"blocked"
|
||||
@@ -106,9 +207,27 @@ fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) {
|
||||
};
|
||||
metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED).set(recovery_required);
|
||||
metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT).set(status.retry_count as f64);
|
||||
*SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner()) = status;
|
||||
*current = status;
|
||||
let next = version.saturating_add(1);
|
||||
SCANNER_CYCLE_RECOVERY_STATUS_VERSION.store(next, Ordering::Relaxed);
|
||||
Some(next)
|
||||
}
|
||||
|
||||
fn publish_scanner_cleanup_failure(reason: String, expected: u64) {
|
||||
let mut status = {
|
||||
let current = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed) != expected {
|
||||
return;
|
||||
}
|
||||
current.clone()
|
||||
};
|
||||
// Preserve the latest core progress, including a newly written primary's
|
||||
// revision and epoch. The original marker may predate that durable write.
|
||||
status.reason = Some(reason);
|
||||
status.last_attempt_at_unix_secs = Some(unix_now_secs());
|
||||
let _ = publish_scanner_cleanup_status(status, Some(expected));
|
||||
}
|
||||
|
||||
pub(super) fn record_scanner_usage_floor_failure(reason: String) {
|
||||
@@ -932,10 +1051,81 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset a blocked cycle state after an operator has explicitly requested a full
|
||||
/// usage rebuild. The primary object is changed first with its observed ETag;
|
||||
/// the recovery marker is removed only when its own ETag still matches.
|
||||
/// Reset a blocked cycle state after an explicit full-rescan request. Durable
|
||||
/// cleanup state fences primary rewrites; marker removal retains its ETag and
|
||||
/// usage-epoch checks.
|
||||
pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
|
||||
reset_scanner_cycle_recovery_for_intent(ctx, storeapi, None, None).await
|
||||
}
|
||||
|
||||
/// Resume only an operator reset whose cleanup phase is already durable.
|
||||
/// Missing or merely blocked markers never authorize an automatic reset.
|
||||
pub(super) async fn resume_scanner_cycle_cleanup(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
|
||||
let status_version = scanner_cleanup_status_version();
|
||||
let (marker, revision) = match read_scanner_cleanup_marker(storeapi.clone(), &ctx).await {
|
||||
Ok(Some(marker)) => marker,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(error) => {
|
||||
let _ =
|
||||
publish_scanner_cleanup_status(recovery_status("blocked", Some(&error.to_string()), false), Some(status_version));
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let mut observation =
|
||||
publish_scanner_cleanup_status(recovery_status_from_marker(&marker, &marker.state), Some(status_version));
|
||||
if marker.state != "cleanup-pending" {
|
||||
return Ok(());
|
||||
}
|
||||
let result = reset_scanner_cycle_recovery_for_intent(ctx, storeapi, Some(revision), Some(&mut observation)).await;
|
||||
if let Err(error) = &result
|
||||
&& let Some(observation) = observation
|
||||
{
|
||||
publish_scanner_cleanup_failure(error.to_string(), observation);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn read_scanner_cleanup_marker(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
ctx: &CancellationToken,
|
||||
) -> Result<Option<(ScannerCycleRecoveryMarker, DataUsageCacheRevision)>, ScannerError> {
|
||||
let (data, revision) = tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => return Err(ScannerError::Other("scanner cleanup recovery was cancelled".to_string())),
|
||||
result = tokio::time::timeout(data_usage_persist_timeout(), read_cycle_recovery_marker_bytes(storeapi)) => {
|
||||
result.map_err(|_| ScannerError::Other("scanner cleanup marker inspection timed out".to_string()))?
|
||||
.map_err(|err| ScannerError::Other(format!("failed to inspect pending scanner cleanup: {err}")))?
|
||||
}
|
||||
};
|
||||
let Some(data) = data else {
|
||||
return Ok(None);
|
||||
};
|
||||
let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&data)
|
||||
.map_err(|err| ScannerError::Other(format!("pending scanner cleanup marker is invalid: {err}")))?;
|
||||
validate_recovery_marker(&marker)
|
||||
.map_err(|err| ScannerError::Other(format!("pending scanner cleanup marker is invalid: {err}")))?;
|
||||
Ok(Some((marker, revision)))
|
||||
}
|
||||
|
||||
pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
expected_cleanup_revision: Option<DataUsageCacheRevision>,
|
||||
mut observation: Option<&mut Option<u64>>,
|
||||
) -> Result<(), ScannerError> {
|
||||
#[cfg(test)]
|
||||
let resume_only = expected_cleanup_revision.is_some();
|
||||
// The outer Some distinguishes a tracked resume whose version may be
|
||||
// invalidated from an explicit v3 reset with no observation owner.
|
||||
let mut publish_status = |status| {
|
||||
if let Some(version) = observation.as_deref_mut() {
|
||||
if let Some(expected) = *version {
|
||||
*version = publish_scanner_cleanup_status(status, Some(expected));
|
||||
}
|
||||
} else {
|
||||
set_scanner_cycle_recovery_status(status);
|
||||
}
|
||||
};
|
||||
let lock = storeapi
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
@@ -964,6 +1154,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
|
||||
};
|
||||
if let Some(expected) = expected_cleanup_revision {
|
||||
if marker_revision != expected || !owns_reset() {
|
||||
return Err(ScannerError::Other(
|
||||
"pending scanner cleanup changed before recovery acquired ownership".to_string(),
|
||||
));
|
||||
}
|
||||
let marker: ScannerCycleRecoveryMarker = marker_data
|
||||
.as_deref()
|
||||
.and_then(|data| serde_json::from_slice(data).ok())
|
||||
.filter(|marker| validate_recovery_marker(marker).is_ok() && marker.state == "cleanup-pending")
|
||||
.ok_or_else(|| {
|
||||
ScannerError::Other("scanner cleanup recovery requires an unchanged cleanup-pending marker".to_string())
|
||||
})?;
|
||||
publish_status(recovery_status_from_marker(&marker, "cleanup-pending"));
|
||||
}
|
||||
let Some(marker_data) = marker_data else {
|
||||
// A delete may commit before its reply is lost. Confirm both durable
|
||||
// fences before treating a retry without its marker as completed.
|
||||
@@ -981,7 +1186,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
|
||||
));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
publish_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
};
|
||||
@@ -997,6 +1202,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::PrimaryRead, owns_reset())?;
|
||||
}
|
||||
let (mut primary_reader, primary_revision) = match storeapi
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -1062,7 +1271,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
let (cleanup_marker, cleanup_marker_revision) =
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
|
||||
.await?;
|
||||
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
publish_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
let fence_epoch = primary_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
@@ -1082,6 +1291,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
#[cfg(test)]
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::PrimaryWrite, owns_reset())?;
|
||||
}
|
||||
let preserved_info = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
@@ -1155,7 +1368,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
|
||||
}
|
||||
})?;
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
publish_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
} else if !force_full_rescan && !marker_cleanup_pending {
|
||||
@@ -1228,11 +1441,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
if let Err(err) =
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
|
||||
.await
|
||||
{
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
let usage_fence = fence_scanner_usage_epoch_with_expected_epoch(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
leader_epoch,
|
||||
Some(reset_epoch),
|
||||
false,
|
||||
&owns_reset,
|
||||
);
|
||||
#[cfg(test)]
|
||||
let usage_fence = async {
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::UsageFence, owns_reset())?;
|
||||
}
|
||||
usage_fence.await
|
||||
};
|
||||
if let Err(err) = usage_fence.await {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1260,7 +1485,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
};
|
||||
if current_revision != rebuilt_revision {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1281,7 +1506,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1318,7 +1543,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
.await
|
||||
{
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1336,7 +1561,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner recovery reset deferred by a movement epoch change".to_string(),
|
||||
));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1352,7 +1577,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
});
|
||||
return Err(ScannerError::Other(format!("failed to clear cycle recovery marker: {err}")));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "healthy".to_string(),
|
||||
|
||||
@@ -27,6 +27,7 @@ use crate::{
|
||||
use serial_test::serial;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::task::Poll;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
@@ -36,6 +37,11 @@ use tokio::time::{Duration, advance};
|
||||
|
||||
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
|
||||
mod quota_reset_preservation;
|
||||
|
||||
mod recovery_control;
|
||||
mod scoped_ack_publication;
|
||||
|
||||
async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
setup_scanner_cycle_store_with_usage_baseline(true).await
|
||||
}
|
||||
@@ -50,11 +56,17 @@ async fn setup_scanner_cycle_store_with_pool_count(
|
||||
) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let store = setup_scanner_cycle_store_at_path(temp_dir.path(), seed_usage_baseline, pool_count).await;
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
async fn setup_scanner_cycle_store_at_path(root: &Path, seed_usage_baseline: bool, pool_count: usize) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}"));
|
||||
let disk_path = root.join(format!("pool{pool_index}/disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("scanner cycle test disk should be created");
|
||||
@@ -104,7 +116,7 @@ async fn setup_scanner_cycle_store_with_pool_count(
|
||||
.expect("scanner cycle usage baseline should persist");
|
||||
}
|
||||
|
||||
(temp_dir, store)
|
||||
store
|
||||
}
|
||||
|
||||
async fn restart_scanner_cycle_store_from(store: &Arc<ECStore>) -> Arc<ECStore> {
|
||||
@@ -1219,7 +1231,18 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget), true),
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
1,
|
||||
Arc::clone(&budget),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("the coordinator must finish its namespace walk while a PUT is pending");
|
||||
@@ -1263,7 +1286,18 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget), true),
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
1,
|
||||
Arc::clone(&retry_budget),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("the same cycle must converge after the pending PUT drains");
|
||||
@@ -5264,6 +5298,103 @@ async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewrit
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_resumes_real_store_cleanup_boundaries_after_reopen() {
|
||||
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let cleanup_paths = [
|
||||
format!("{primary_path}.bkp"),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
|
||||
];
|
||||
|
||||
for completed in 0..=cleanup_paths.len() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 42,
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
encode_scanner_cycle_state(&cycle, 3).expect("cycle state should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("cycle state should persist");
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
|
||||
save_config(
|
||||
store.clone(),
|
||||
primary_path,
|
||||
serde_json::to_vec(&marker).expect("usage reset marker should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("usage reset marker should persist");
|
||||
|
||||
for path in cleanup_paths.iter().skip(completed) {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
usage.scanner_cycle = Some(12);
|
||||
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("cleanup slot should encode"))
|
||||
.await
|
||||
.expect("cleanup slot should persist");
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
save_config(store.clone(), path, b"retain".to_vec())
|
||||
.await
|
||||
.expect("unrelated state should persist before reopen");
|
||||
}
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let intent_before = read_config_with_revision(restarted.clone(), primary_path)
|
||||
.await
|
||||
.expect("reopened reset intent should be readable");
|
||||
|
||||
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), restarted.clone())
|
||||
.await
|
||||
.expect("reopened usage reset should complete");
|
||||
|
||||
assert_eq!(result.leader_epoch, 3, "boundary {completed}");
|
||||
assert_eq!(result.next_cycle, 42, "boundary {completed}");
|
||||
assert_eq!(result.reset_paths.len(), cleanup_paths.len() + 1 - completed, "boundary {completed}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(restarted.clone(), primary_path)
|
||||
.await
|
||||
.expect("completed reset intent should remain readable"),
|
||||
intent_before,
|
||||
"boundary {completed}: resumed cleanup must not rewrite the reset intent"
|
||||
);
|
||||
|
||||
let (floor, state) = persisted_usage_floor_for_startup(restarted.clone(), false)
|
||||
.await
|
||||
.expect("completed reset marker should remain resumable");
|
||||
assert_eq!(floor.leader_epoch, 3, "boundary {completed}");
|
||||
assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending, "boundary {completed}");
|
||||
assert!(
|
||||
persisted_usage_floor(restarted.clone()).await.is_err(),
|
||||
"boundary {completed}: bootstrap marker must not become an authoritative floor"
|
||||
);
|
||||
|
||||
for path in &cleanup_paths {
|
||||
assert!(
|
||||
matches!(read_config(restarted.clone(), path).await, Err(EcstoreError::ConfigNotFound)),
|
||||
"boundary {completed}: reset should remove stale usage slot {path}"
|
||||
);
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
assert_eq!(
|
||||
read_config(restarted.clone(), path)
|
||||
.await
|
||||
.expect("unrelated state should survive reopened reset"),
|
||||
b"retain",
|
||||
"boundary {completed}: reset must preserve non-scanner-state config"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -6160,7 +6291,7 @@ async fn coordinator_classifies_an_expired_publication_lease() {
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
outcome,
|
||||
outcome.outcome(),
|
||||
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
|
||||
);
|
||||
assert!(store.put_counts.lock().await.is_empty(), "expired lease must prevent a PUT");
|
||||
@@ -7301,7 +7432,7 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
fn finalizing_a_saved_enum_without_proof_keeps_dirty_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
@@ -7309,21 +7440,23 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
let remote_acknowledgement = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 11,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
};
|
||||
let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone()))
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate.into());
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Failed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
|
||||
let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot))
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved);
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement]);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved.into());
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert_eq!(acknowledgements, vec![remote_acknowledgement]);
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(pending);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -7335,7 +7468,7 @@ fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
|
||||
let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
|
||||
let (outcome, _, acknowledgements) =
|
||||
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement).into());
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7355,7 +7488,7 @@ fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
|
||||
)
|
||||
.with_observational_snapshot_published(true);
|
||||
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved.into());
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Partial);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7391,17 +7524,20 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
fn finalizing_an_already_durable_enum_without_proof_keeps_dirty_pending() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (outcome, pending, acknowledgements) =
|
||||
finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable.into());
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
assert!(pending);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -7412,7 +7548,8 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable);
|
||||
let (outcome, _, acknowledgements) =
|
||||
finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable.into());
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7428,7 +7565,7 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot));
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved.into());
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Superseded);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -8514,6 +8651,56 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn service_cohort_aging_preserves_explicit_cycle_wait() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(3600),
|
||||
cycle_interval_source: ScannerRuntimeConfigSource::Env,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = ScannerCycleObservedGenerations::for_wait(
|
||||
&config,
|
||||
None,
|
||||
crate::scanner_io::dirty_usage_generation(),
|
||||
crate::runtime_config::scanner_runtime_config_generation(),
|
||||
crate::scanner_io::scanner_maintenance_generation(),
|
||||
);
|
||||
assert_eq!(observed.dirty_usage, None);
|
||||
let inventory = HashMap::from([(
|
||||
crate::data_usage_define::DataUsageCacheSource::new(0, 0),
|
||||
vec![crate::storage_api::scanner_io::BucketInfo {
|
||||
name: "waiting-bootstrap".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
)]);
|
||||
let mut cohort = crate::scanner_io::ScannerServiceCohort::default();
|
||||
cohort.refresh(&inventory);
|
||||
let ctx = CancellationToken::new();
|
||||
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
||||
&ctx,
|
||||
config.cycle_interval,
|
||||
observed.dirty_usage,
|
||||
observed.runtime_config,
|
||||
observed.maintenance,
|
||||
|| false,
|
||||
));
|
||||
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
||||
for _ in 0..59 {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
cohort.refresh(&inventory);
|
||||
crate::scanner_io::record_dirty_usage_bucket("hot");
|
||||
assert!(
|
||||
matches!(futures::poll!(&mut wait), Poll::Pending),
|
||||
"aging/dirty must not shorten the explicit hour"
|
||||
);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(wait.await, ScannerCycleWakeReason::Timer);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
||||
@@ -8802,12 +8989,12 @@ fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acqui
|
||||
ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
generation: 5,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
|
||||
},
|
||||
]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
|
||||
result,
|
||||
DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")),
|
||||
DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")).into(),
|
||||
);
|
||||
assert_eq!(
|
||||
outcome,
|
||||
@@ -8962,7 +9149,7 @@ fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
||||
vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-3".to_string(),
|
||||
instance_id: "epoch-dirty".to_string(),
|
||||
generation: 11,
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::owner::ObjectOperations as _;
|
||||
|
||||
const BUCKET: &str = "quota-reset-preservation";
|
||||
const OPERATION: &str = "00000000-0000-0000-0000-000000000002";
|
||||
|
||||
async fn reservation_fixture() -> (tempfile::TempDir, Arc<ECStore>, Uuid, String, Vec<u8>) {
|
||||
let (directory, store) = setup_scanner_cycle_store().await;
|
||||
store
|
||||
.make_bucket(BUCKET, &crate::storage_api::scan::MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the reservation fixture bucket through its owner");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("durable bucket incarnation");
|
||||
assert!(!incarnation.is_nil());
|
||||
let path = format!("config/quota-ledger/{BUCKET}.json");
|
||||
let bytes = serde_json::to_vec(&serde_json::json!({
|
||||
"version": 1,
|
||||
"bucket_incarnation": incarnation,
|
||||
"quota_revision_unix_nanos": 1,
|
||||
"accounted_usage": 100,
|
||||
"reservations": {
|
||||
OPERATION: {
|
||||
"object": "pending-object",
|
||||
"old_size": 0,
|
||||
"new_size": 64,
|
||||
"created_at": 1,
|
||||
"pool_index": 0,
|
||||
"set_index": 0,
|
||||
"commit_started": true
|
||||
}
|
||||
}
|
||||
}))
|
||||
.expect("encode the committed reservation fixture");
|
||||
save_config(store.clone(), &path, bytes.clone())
|
||||
.await
|
||||
.expect("persist reservation bytes through the real storage owner");
|
||||
(directory, store, incarnation, path, bytes)
|
||||
}
|
||||
|
||||
async fn assert_reservation_retained(store: &Arc<ECStore>, path: &str, expected: &[u8], incarnation: Uuid) {
|
||||
let bytes = read_config(store.clone(), path)
|
||||
.await
|
||||
.expect("read the actual reservation ledger");
|
||||
assert_eq!(bytes, expected, "scanner reset must not rewrite the reservation ledger");
|
||||
let ledger: serde_json::Value = serde_json::from_slice(&bytes).expect("persisted ledger JSON");
|
||||
assert_eq!(ledger["version"], 1);
|
||||
assert_eq!(ledger["bucket_incarnation"], incarnation.to_string());
|
||||
assert_eq!(ledger["accounted_usage"], 100);
|
||||
let reservations = ledger["reservations"].as_object().expect("reservation map");
|
||||
assert_eq!(reservations.len(), 1);
|
||||
let pending = &reservations[OPERATION];
|
||||
assert_eq!(pending["old_size"], 0);
|
||||
assert_eq!(pending["new_size"], 64);
|
||||
assert_eq!(pending["commit_started"], true);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("owner incarnation after restart"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reset_preservation_survives_storage_owner_reconstruction() {
|
||||
let (_directory, store, incarnation, path, bytes) = reservation_fixture().await;
|
||||
let reset = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("reset scanner usage through the fenced production entry");
|
||||
assert_eq!(reset.usage_state, "bootstrap-pending");
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
assert!(
|
||||
!Arc::ptr_eq(&store, &restarted),
|
||||
"the assertion must read through a newly constructed ECStore"
|
||||
);
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read reset usage through the reconstructed owner");
|
||||
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("bootstrap usage JSON");
|
||||
assert!(data_usage_info_is_bootstrap_pending(&usage));
|
||||
assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reset_preservation_unknown_protocol_rejects_put_after_restart() {
|
||||
for quota_shape in ["zero", "null", "missing"] {
|
||||
let (_directory, store, incarnation, path, bytes) = reservation_fixture().await;
|
||||
let mut quota = serde_json::json!({
|
||||
"quota_type": "Hard",
|
||||
"reservation_protocol": 2,
|
||||
"reservation_quota": 1024
|
||||
});
|
||||
match quota_shape {
|
||||
"zero" => quota["quota"] = serde_json::json!(0),
|
||||
"null" => quota["quota"] = serde_json::Value::Null,
|
||||
"missing" => {}
|
||||
_ => unreachable!("fixed quota shapes"),
|
||||
}
|
||||
let unknown_quota = serde_json::to_vec("a).expect("unknown but syntactically valid quota protocol");
|
||||
store
|
||||
.update_bucket_metadata_config(BUCKET, rustfs_config::QUOTA_CONFIG_FILE, unknown_quota)
|
||||
.await
|
||||
.expect("persist a future protocol using the real metadata owner");
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("same metadata owner incarnation"),
|
||||
incarnation
|
||||
);
|
||||
reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("scanner reset must not change quota metadata");
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
assert!(!Arc::ptr_eq(&store, &restarted));
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
let mut reader = PutObjReader::from_vec(b"must-not-commit".to_vec());
|
||||
let result = restarted.pools[0].disk_set[0]
|
||||
.put_object(BUCKET, "rejected-object", &mut reader, &ObjectOptions::default())
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("unknown reservation protocol with quota={quota_shape} must not admit a PUT"),
|
||||
};
|
||||
assert!(
|
||||
matches!(error, EcstoreError::PartMissingOrCorrupt),
|
||||
"unexpected protocol rejection: {error}"
|
||||
);
|
||||
let missing = restarted.pools[0].disk_set[0]
|
||||
.get_object_info(BUCKET, "rejected-object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the rejected PUT must not create an object");
|
||||
assert!(
|
||||
matches!(missing, EcstoreError::FileNotFound | EcstoreError::ObjectNotFound(_, _)),
|
||||
"object absence must not be confused with another storage failure: {missing}"
|
||||
);
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::super::cycle_state::cleanup_io_fault;
|
||||
use super::*;
|
||||
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
|
||||
|
||||
async fn seed_cleanup(store: &Arc<ECStore>, state: &str) -> ScannerCycleRecoveryMarker {
|
||||
let cycle = CurrentCycle {
|
||||
current: 3,
|
||||
next: 42,
|
||||
..Default::default()
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
encode_scanner_cycle_state(&cycle, 7).expect("cycle encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist cycle");
|
||||
let usage = DataUsageInfo {
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(41),
|
||||
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("usage encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist usage floor");
|
||||
let marker = ScannerCycleRecoveryMarker {
|
||||
schema_version: 1,
|
||||
primary_revision: "previous-primary".to_string(),
|
||||
generation: 41,
|
||||
leader_epoch: 7,
|
||||
classification: "corrupt".to_string(),
|
||||
first_detected_at_unix_secs: 1,
|
||||
last_attempt_at_unix_secs: 2,
|
||||
retry_count: 1,
|
||||
reason: "operator reset in progress".to_string(),
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
|
||||
state: state.to_string(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
serde_json::to_vec(&marker).expect("marker encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist operator marker");
|
||||
marker
|
||||
}
|
||||
|
||||
async fn persisted_state(store: &Arc<ECStore>) -> Vec<(Option<Vec<u8>>, DataUsageCacheRevision)> {
|
||||
let mut state = Vec::new();
|
||||
for path in [
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
] {
|
||||
state.push(
|
||||
read_config_with_revision(store.clone(), path)
|
||||
.await
|
||||
.expect("read exact metadata revision"),
|
||||
);
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
async fn run_disabled_startup(ctx: CancellationToken, store: Arc<ECStore>) {
|
||||
let initialized_before = crate::scanner_runtime_initialized();
|
||||
let cleanup = init_scanner_with_recovery(ctx, store, false).await;
|
||||
if let Some(cleanup) = cleanup {
|
||||
tokio::time::timeout(Duration::from_secs(15), cleanup)
|
||||
.await
|
||||
.expect("finite disabled cleanup attempt")
|
||||
.expect("cleanup task should not panic");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::scanner_runtime_initialized(),
|
||||
initialized_before,
|
||||
"disabled recovery must not start the normal scanner runtime"
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_reset_fences(store: &Arc<ECStore>) {
|
||||
let data = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle remains durable");
|
||||
let (cycle, epoch) = decode_scanner_cycle_state(&data).expect("valid preserved cycle");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (3, 42, 8));
|
||||
let usage: DataUsageInfo = serde_json::from_slice(
|
||||
&read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("durable usage fence"),
|
||||
)
|
||||
.expect("valid usage");
|
||||
assert_eq!(usage.scanner_epoch, Some(8));
|
||||
assert_eq!(usage.scanner_cycle, Some(41));
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
async fn assert_rebuilt_reset_fences(store: &Arc<ECStore>, expected_epoch: u64) {
|
||||
let data = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt cycle remains durable");
|
||||
let (cycle, epoch) = decode_scanner_cycle_state(&data).expect("valid rebuilt cycle");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (0, 42, expected_epoch));
|
||||
let usage: DataUsageInfo = serde_json::from_slice(
|
||||
&read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("durable rebuilt usage fence"),
|
||||
)
|
||||
.expect("valid usage");
|
||||
assert_eq!(usage.scanner_epoch, Some(expected_epoch));
|
||||
assert_eq!(usage.scanner_cycle, Some(41));
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_reset_crash_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_SCANNER_RESET_CRASH_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let stage = match std::env::var("RUSTFS_SCANNER_RESET_CRASH_STAGE").as_deref() {
|
||||
Ok("primary-read") => cleanup_io_fault::Stage::PrimaryRead,
|
||||
Ok("primary-write") => cleanup_io_fault::Stage::PrimaryWrite,
|
||||
Ok("usage-fence") => cleanup_io_fault::Stage::UsageFence,
|
||||
other => panic!("unexpected reset crash stage: {other:?}"),
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let store = setup_scanner_cycle_store_at_path(std::path::Path::new(&root), false, 1).await;
|
||||
if stage == cleanup_io_fault::Stage::UsageFence {
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt-cycle".to_vec())
|
||||
.await
|
||||
.expect("force child through reconstruction branch");
|
||||
}
|
||||
let injection = cleanup_io_fault::install(&store, stage, false);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store)
|
||||
.await
|
||||
.expect_err("child should stop at the injected owned I/O boundary");
|
||||
assert!(injection.fired_while_owned());
|
||||
let expected = match stage {
|
||||
cleanup_io_fault::Stage::PrimaryRead => "injected primary read failure",
|
||||
cleanup_io_fault::Stage::PrimaryWrite => "injected primary write failure",
|
||||
cleanup_io_fault::Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
assert!(error.to_string().contains(expected), "{error}");
|
||||
});
|
||||
std::process::exit(77);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_recovers_after_child_process_crash_boundaries() {
|
||||
for (name, expected_rebuilt_epoch) in [("primary-read", None), ("primary-write", None), ("usage-fence", Some(9))] {
|
||||
let temp_dir = tempfile::tempdir().expect("crash fixture directory");
|
||||
let store = setup_scanner_cycle_store_at_path(temp_dir.path(), false, 1).await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
drop(store);
|
||||
|
||||
let status = std::process::Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("scanner::tests::recovery_control::scanner_reset_crash_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_SCANNER_RESET_CRASH_ROOT", temp_dir.path())
|
||||
.env("RUSTFS_SCANNER_RESET_CRASH_STAGE", name)
|
||||
.status()
|
||||
.expect("child crash fixture should start");
|
||||
assert_eq!(status.code(), Some(77), "{name} child did not reach the owned crash boundary");
|
||||
|
||||
let restarted = setup_scanner_cycle_store_at_path(temp_dir.path(), false, 1).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
if let Some(epoch) = expected_rebuilt_epoch {
|
||||
assert_rebuilt_reset_fences(&restarted, epoch).await;
|
||||
} else {
|
||||
assert_reset_fences(&restarted).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_reopens_persisted_intent_without_starting_scanner() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
assert_reset_fences(&restarted).await;
|
||||
let completed = persisted_state(&restarted).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
assert_eq!(
|
||||
persisted_state(&restarted).await,
|
||||
completed,
|
||||
"a later startup without an intent must not reset again"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_does_not_authorize_blocked_unknown_or_corrupt_markers() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
for kind in ["blocked", "unknown-phase", "future-version", "unknown-field", "corrupt"] {
|
||||
let marker = seed_cleanup(&store, "blocked").await;
|
||||
let mut value = serde_json::to_value(marker).expect("marker value");
|
||||
match kind {
|
||||
"unknown-phase" => value["state"] = "future-phase".into(),
|
||||
"future-version" => value["schema_version"] = 99.into(),
|
||||
"unknown-field" => value["future_hint"] = true.into(),
|
||||
_ => {}
|
||||
}
|
||||
let bytes = if kind == "corrupt" {
|
||||
b"{broken".to_vec()
|
||||
} else {
|
||||
serde_json::to_vec(&value).expect("marker JSON")
|
||||
};
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes)
|
||||
.await
|
||||
.expect("persist rejected marker");
|
||||
let before = persisted_state(&store).await;
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_eq!(persisted_state(&store).await, before, "{kind} must not become an automatic full rescan");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_rechecks_revision_after_waiting_for_leader_lock() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let mut marker = seed_cleanup(&store, "cleanup-pending").await;
|
||||
let expected = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("intent revision")
|
||||
.1;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold leader ownership");
|
||||
let mut recovery = Box::pin(reset_scanner_cycle_recovery_for_intent(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
Some(expected),
|
||||
None,
|
||||
));
|
||||
assert!(matches!(futures::poll!(&mut recovery), Poll::Pending));
|
||||
marker.state = "blocked".to_string();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
serde_json::to_vec(&marker).expect("replacement marker"),
|
||||
)
|
||||
.await
|
||||
.expect("replace intent while the fixture owns leader lock");
|
||||
let replaced = persisted_state(&store).await;
|
||||
drop(guard);
|
||||
let error = recovery.await.expect_err("old preflight cannot authorize replacement marker");
|
||||
assert!(error.to_string().contains("changed before recovery acquired ownership"));
|
||||
assert_eq!(persisted_state(&store).await, replaced);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_requires_phase_even_when_revision_matches() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "blocked").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let error = reset_scanner_cycle_recovery_for_intent(CancellationToken::new(), store.clone(), Some(before[1].1.clone()), None)
|
||||
.await
|
||||
.expect_err("a matching ETag alone is not operator cleanup authorization");
|
||||
assert!(error.to_string().contains("unchanged cleanup-pending"));
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("explicit v3 core retains full reset authorization");
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_lock_busy_preserves_intent_without_force_unlock() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader");
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("busy leader must block recovery");
|
||||
assert!(error.to_string().contains("leader lock is busy"));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("leader lock is busy"))
|
||||
);
|
||||
assert!(!status.retryable, "disabled startup makes one attempt, not an automatic retry loop");
|
||||
assert!(!guard.is_lock_lost(), "recovery must not revoke the live owner");
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
drop(guard);
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn concurrent_full_rescan_requests_converge_to_one_recovery_fence() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "blocked").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader before concurrent admin resets");
|
||||
|
||||
let first = tokio::spawn(reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()));
|
||||
let second = tokio::spawn(reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()));
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
persisted_state(&store).await,
|
||||
before,
|
||||
"waiting admin reset requests must not mutate state before leader ownership"
|
||||
);
|
||||
|
||||
drop(guard);
|
||||
let (first, second) = tokio::time::timeout(Duration::from_secs(15), async { tokio::join!(first, second) })
|
||||
.await
|
||||
.expect("both admin reset requests should finish after the live owner releases the lock");
|
||||
let first = first.expect("first admin reset task should not panic");
|
||||
let second = second.expect("second admin reset task should not panic");
|
||||
assert!(first.is_ok(), "first admin reset failed: {first:?}");
|
||||
assert!(second.is_ok(), "second admin reset failed: {second:?}");
|
||||
|
||||
assert_reset_fences(&store).await;
|
||||
let completed = persisted_state(&store).await;
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("lost-reply retry after a completed reset should be idempotent");
|
||||
assert_eq!(
|
||||
persisted_state(&store).await,
|
||||
completed,
|
||||
"a later retry must not create a second reset identity or rewrite durable fences"
|
||||
);
|
||||
assert_eq!(scanner_cycle_recovery_status().state, "healthy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_movement_pause_preserves_intent_for_later_startup() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
|
||||
id: "cleanup-movement".to_string(),
|
||||
pool_stats: vec![EcstoreRebalanceStats {
|
||||
participating: true,
|
||||
info: EcstoreRebalanceInfo {
|
||||
start_time: Some(time::OffsetDateTime::now_utc()),
|
||||
status: EcstoreRebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("movement must block reset publication");
|
||||
assert!(error.to_string().contains("blocked by data movement"));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("blocked by data movement"))
|
||||
);
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
*store.rebalance_meta.write().await = None;
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_cancelled_startup_preserves_persisted_work() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let ctx = CancellationToken::new();
|
||||
ctx.cancel();
|
||||
run_disabled_startup(ctx, store.clone()).await;
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_probe_obeys_cancellation_and_existing_io_deadline() {
|
||||
for cancel in [false, true] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
store.delayed_gets.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()),
|
||||
data_usage_persist_timeout().saturating_add(Duration::from_secs(60)),
|
||||
);
|
||||
let ctx = CancellationToken::new();
|
||||
let mut probe = Box::pin(read_scanner_cleanup_marker(store.clone(), &ctx));
|
||||
assert!(matches!(futures::poll!(&mut probe), Poll::Pending));
|
||||
if cancel {
|
||||
ctx.cancel();
|
||||
} else {
|
||||
advance(data_usage_persist_timeout()).await;
|
||||
}
|
||||
let error = probe.await.expect_err("pending read must be bounded");
|
||||
assert!(error.to_string().contains(if cancel { "cancelled" } else { "timed out" }));
|
||||
assert!(store.put_counts.lock().await.is_empty(), "probe must remain read-only");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn disabled_cleanup_old_observation_cannot_overwrite_a_new_completion() {
|
||||
let original = scanner_cycle_recovery_status();
|
||||
let healthy = ScannerCycleRecoveryStatus {
|
||||
state: "healthy".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
publish_scanner_cleanup_status(healthy.clone(), None).expect("first status version");
|
||||
let old = scanner_cleanup_status_version();
|
||||
publish_scanner_cleanup_status(healthy, None).expect("a newer completion may have identical fields");
|
||||
assert!(
|
||||
publish_scanner_cleanup_status(
|
||||
ScannerCycleRecoveryStatus {
|
||||
state: "cleanup-pending".to_string(),
|
||||
reason: Some("old lock wait failed".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(old)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(scanner_cycle_recovery_status().state, "healthy");
|
||||
publish_scanner_cleanup_status(original, None).expect("restore prior observation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_owned_read_and_write_failures_keep_specific_status() {
|
||||
for (stage, newer_completion) in [
|
||||
(cleanup_io_fault::Stage::PrimaryRead, false),
|
||||
(cleanup_io_fault::Stage::PrimaryWrite, false),
|
||||
(cleanup_io_fault::Stage::PrimaryWrite, true),
|
||||
] {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let injection = cleanup_io_fault::install(&store, stage, newer_completion);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("injected owned I/O boundary");
|
||||
assert!(
|
||||
injection.fired_while_owned(),
|
||||
"fault must occur after real leader ownership and marker validation"
|
||||
);
|
||||
let expected_error = match stage {
|
||||
cleanup_io_fault::Stage::PrimaryRead => "injected primary read failure",
|
||||
cleanup_io_fault::Stage::PrimaryWrite => "injected primary write failure",
|
||||
cleanup_io_fault::Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
assert!(error.to_string().contains(expected_error));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if newer_completion {
|
||||
assert_eq!(status.state, "healthy", "old error cannot overwrite a newer completion observation");
|
||||
assert!(status.reason.is_none());
|
||||
} else {
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status.reason.as_deref().is_some_and(|reason| reason.contains(expected_error)),
|
||||
"{status:?}"
|
||||
);
|
||||
}
|
||||
let after = persisted_state(&store).await;
|
||||
assert_eq!(after[0], before[0], "failed primary I/O must preserve its prior revision");
|
||||
assert_eq!(after[2], before[2], "failed primary I/O must not advance the usage fence");
|
||||
let marker: ScannerCycleRecoveryMarker =
|
||||
serde_json::from_slice(after[1].0.as_deref().expect("durable marker retained")).expect("valid cleanup marker");
|
||||
assert_eq!(marker.state, "cleanup-pending");
|
||||
drop(injection);
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_invalidated_observation_never_becomes_unconditional() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let revision = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker revision")
|
||||
.1;
|
||||
let newer = ScannerCycleRecoveryStatus {
|
||||
state: "healthy".to_string(),
|
||||
reason: Some("newer completion owner".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
publish_scanner_cleanup_status(newer.clone(), None).expect("newer observation");
|
||||
let mut invalidated = None;
|
||||
reset_scanner_cycle_recovery_for_intent(CancellationToken::new(), store.clone(), Some(revision), Some(&mut invalidated))
|
||||
.await
|
||||
.expect("metadata cleanup may complete without owning the newest status observation");
|
||||
assert!(invalidated.is_none());
|
||||
assert_eq!(
|
||||
serde_json::to_value(scanner_cycle_recovery_status()).expect("observed status"),
|
||||
serde_json::to_value(newer).expect("newer status")
|
||||
);
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_later_failure_preserves_rebuilt_primary_status_identity() {
|
||||
for newer_completion in [false, true] {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt-cycle".to_vec())
|
||||
.await
|
||||
.expect("force the full reconstruction branch");
|
||||
let before = persisted_state(&store).await;
|
||||
let injection = cleanup_io_fault::install(&store, cleanup_io_fault::Stage::UsageFence, newer_completion);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("fail after primary publication");
|
||||
assert!(injection.fired_while_owned());
|
||||
assert!(error.to_string().contains("injected usage fence failure"));
|
||||
let after = persisted_state(&store).await;
|
||||
assert_ne!(after[0].1, before[0].1, "the primary write must actually commit before this failure");
|
||||
let (cycle, epoch) =
|
||||
decode_scanner_cycle_state(after[0].0.as_deref().expect("rebuilt primary")).expect("valid durable reconstruction");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (0, 42, 8));
|
||||
assert_eq!(after[2], before[2], "usage fence publication was rejected");
|
||||
let marker: ScannerCycleRecoveryMarker =
|
||||
serde_json::from_slice(after[1].0.as_deref().expect("cleanup marker retained")).expect("valid cleanup marker");
|
||||
assert_eq!(marker.state, "cleanup-pending");
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if newer_completion {
|
||||
assert_eq!(status.state, "healthy");
|
||||
assert!(
|
||||
status.reason.is_none(),
|
||||
"old core and outer failure must both retain invalidated ownership"
|
||||
);
|
||||
} else {
|
||||
let DataUsageCacheRevision::Etag(etag) = &after[0].1 else {
|
||||
panic!("rebuilt primary must have a revision");
|
||||
};
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert_eq!(status.primary_revision.as_deref(), Some(etag.as_str()));
|
||||
assert_eq!(status.generation, Some(cycle.next));
|
||||
assert_eq!(status.leader_epoch, Some(epoch));
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("injected usage fence failure"))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::super::usage_store::DataUsagePublicationResult;
|
||||
use super::*;
|
||||
use crate::scanner_io::ScannerBucketScanScope;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use sha2::Digest;
|
||||
use std::time::SystemTime;
|
||||
|
||||
const PROOF_BUCKET: &str = "publication-proof-bucket";
|
||||
const PROOF_EPOCH: u64 = 7;
|
||||
const PROOF_CYCLE: u64 = 11;
|
||||
|
||||
async fn settle_namespace_commits(store: &ECStore) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while store.scanner_data_usage_publication_blocked().await {
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fixture namespace commits must settle before collecting complete coverage");
|
||||
}
|
||||
|
||||
async fn complete_candidate(store: &Arc<ECStore>, cycle: u64) -> (crate::scanner_io::ScannerCycleResult, DataUsageInfo) {
|
||||
settle_namespace_commits(store).await;
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let result = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
store.as_ref(),
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx,
|
||||
budget,
|
||||
updates,
|
||||
want_cycle: cycle,
|
||||
leader_epoch: PROOF_EPOCH,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: None,
|
||||
observed_usage_candidate: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
resolved_scope_observer: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real scanner must produce the fixture candidate");
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
let candidate = receiver.recv().await.expect("complete scanner snapshot");
|
||||
assert!(candidate.usage_snapshot_complete);
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(true));
|
||||
assert_eq!(candidate.scanner_cycle, Some(cycle));
|
||||
assert_eq!(candidate.scanner_epoch, Some(PROOF_EPOCH));
|
||||
(result, candidate)
|
||||
}
|
||||
|
||||
async fn candidate_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let (directory, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
store
|
||||
.make_bucket(PROOF_BUCKET, &crate::storage_api::scan::MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create proof fixture bucket through the owner");
|
||||
let mut reader = PutObjReader::from_vec(b"proof".to_vec());
|
||||
store.pools[0].disk_set[0]
|
||||
.put_object(PROOF_BUCKET, "initial", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist fixture object through the owner");
|
||||
crate::scanner_io::record_dirty_usage_bucket(PROOF_BUCKET);
|
||||
settle_namespace_commits(&store).await;
|
||||
(directory, store)
|
||||
}
|
||||
|
||||
async fn read_root(store: &Arc<ECStore>) -> (Option<Vec<u8>>, DataUsageCacheRevision) {
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read actual v2 root bytes and revision")
|
||||
}
|
||||
|
||||
async fn publish_candidate(
|
||||
store: &Arc<ECStore>,
|
||||
scan: &crate::scanner_io::ScannerCycleResult,
|
||||
candidate: DataUsageInfo,
|
||||
baseline: Option<DataUsagePersistBaseline>,
|
||||
) -> DataUsagePublicationResult {
|
||||
let expectation = scan.publication_expectation();
|
||||
assert!(expectation.is_some(), "only a real complete scan may supply the expectation");
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender.send(candidate).await.expect("enqueue the real scan candidate");
|
||||
drop(sender);
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
Some(PROOF_EPOCH),
|
||||
baseline,
|
||||
ScannerPublicationFence::new(scan.publication_epoch(), None, None).with_ack_expectation(expectation),
|
||||
|| async { None },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_companion_only_does_not_authorize_root_ack() {
|
||||
for companion in [
|
||||
format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let bytes = serde_json::to_vec(&candidate).expect("actual candidate JSON");
|
||||
save_config(store.clone(), &companion, bytes.clone())
|
||||
.await
|
||||
.expect("persist the companion on real disks");
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("companion fallback baseline");
|
||||
assert_eq!(baseline.data.as_deref(), Some(bytes.as_slice()));
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
assert_eq!(read_root(&store).await.0, None);
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate, Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert!(pending, "unacknowledged durable companion work must remain pending");
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"a companion is not the v2 root target"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, (None, DataUsageCacheRevision::Missing));
|
||||
assert_eq!(read_config(store.clone(), &companion).await.expect("companion retained"), bytes);
|
||||
}
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_actual_root_readback_accepts_semantic_json_equivalence() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let canonical = serde_json::to_vec(&candidate).expect("candidate encoding");
|
||||
let mut value = serde_json::to_value(&candidate).expect("candidate value");
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("usage object")
|
||||
.insert("fixture_unknown_field".into(), serde_json::json!({"retained": true}));
|
||||
let different_bytes = serde_json::to_vec_pretty(&value).expect("noncanonical primary JSON");
|
||||
assert_ne!(different_bytes, canonical);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(&different_bytes).expect("semantic primary"),
|
||||
candidate
|
||||
);
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), different_bytes.clone())
|
||||
.await
|
||||
.expect("persist actual primary representation");
|
||||
let before = read_root(&store).await;
|
||||
assert!(matches!(&before.1, DataUsageCacheRevision::Etag(etag) if !etag.is_empty()));
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("real primary revision");
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, proof) = publication.into_parts();
|
||||
let proof = proof.expect("actual primary readback must produce its own root proof");
|
||||
let expected = scan.publication_expectation().expect("real scan expectation");
|
||||
let (etag, raw_digest) = proof.verified_version_for(&expected).expect("proof must bind this candidate");
|
||||
let DataUsageCacheRevision::Etag(expected_etag) = &before.1 else { panic!("actual root ETag") };
|
||||
assert_eq!(etag, expected_etag);
|
||||
let expected_digest: [u8; 32] = sha2::Sha256::digest(&different_bytes).into();
|
||||
assert_eq!(
|
||||
*raw_digest, expected_digest,
|
||||
"proof must record actual bytes, not reserialized candidate bytes"
|
||||
);
|
||||
// Obtain another proof through the same real readback path rather than
|
||||
// fabricating a publication result from the inspected proof above.
|
||||
let publication = publish_candidate(&store, &scan, candidate, None).await;
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty(), "the single-node fixture has no remote targets");
|
||||
assert!(
|
||||
!crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
"actual root bytes plus a real revision authorize this scan"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, before, "readback must not rewrite unknown fields or whitespace");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_successful_root_cas_authorizes_its_scan() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
let (bytes, revision) = read_root(&store).await;
|
||||
assert!(matches!(revision, DataUsageCacheRevision::Etag(etag) if !etag.is_empty()));
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(&bytes.expect("actual saved root")).expect("root JSON"),
|
||||
candidate
|
||||
);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(!pending);
|
||||
assert!(acknowledgements.is_empty(), "the single-node fixture has no remote targets");
|
||||
assert!(
|
||||
!crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
"the real root CAS must authorize its matching scan"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_observed_candidate_reuse_requires_a_new_root_proof() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let bootstrap = scanner_usage_bootstrap_marker(SystemTime::now(), Some(PROOF_EPOCH));
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&bootstrap).expect("bootstrap root encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist authoritative bootstrap root");
|
||||
let (prior_scan, mut observed_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
// Seed a complete but unconverged observation from real scanner coverage;
|
||||
// the production writer attaches its authoritative baseline identity.
|
||||
observed_candidate.usage_snapshot_converged = Some(false);
|
||||
let observation = publish_candidate(&store, &prior_scan, observed_candidate, None).await;
|
||||
let (outcome, proof) = observation.into_parts();
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Saved);
|
||||
assert!(proof.is_none(), "an observational write cannot authorize a root ACK");
|
||||
let (root_before, revision_before) = read_root(&store).await;
|
||||
let observed = read_config(store.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read real persisted observation");
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let (observer, selected) = tokio::sync::oneshot::channel();
|
||||
let scan = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
store.as_ref(),
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx,
|
||||
budget,
|
||||
updates,
|
||||
want_cycle: PROOF_CYCLE + 1,
|
||||
leader_epoch: PROOF_EPOCH,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: root_before.clone().map(Bytes::from),
|
||||
observed_usage_candidate: Some(Bytes::from(observed)),
|
||||
requires_full_scan: false,
|
||||
service_cohort: None,
|
||||
resolved_scope_observer: Some(observer),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("observation-backed scope must run through the real scanner");
|
||||
let scope = selected.await.expect("production resolver decision");
|
||||
assert_eq!(scope.selected_buckets_for_tests(), Some(&HashSet::from([PROOF_BUCKET.to_string()])));
|
||||
assert_eq!(scan.status, ScannerCycleStatus::Complete);
|
||||
let expectation = scan.publication_expectation().expect("reused coverage must be revalidated");
|
||||
assert!(
|
||||
!expectation.same_candidate(&prior_scan.publication_expectation().expect("prior real candidate")),
|
||||
"the observation cannot transfer the previous scan's expectation"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, (root_before, revision_before));
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
let candidate = receiver.recv().await.expect("new validated root candidate");
|
||||
assert_eq!(candidate.scanner_cycle, Some(PROOF_CYCLE + 1));
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(true));
|
||||
let publication = publish_candidate(&store, &scan, candidate, None).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(!pending);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_stale_root_cas_keeps_dirty_after_bucket_save() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let mut bucket_cache = DataUsageCache::default();
|
||||
bucket_cache
|
||||
.load(store.pools[0].disk_set[0].clone(), &path_join_buf(&[PROOF_BUCKET, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.expect("real bucket checkpoint must be persisted before root publication");
|
||||
assert!(bucket_cache.info.snapshot_complete);
|
||||
assert_eq!(
|
||||
bucket_cache
|
||||
.checked_flatten(PROOF_BUCKET)
|
||||
.expect("persisted bucket root")
|
||||
.objects,
|
||||
1
|
||||
);
|
||||
let stale_baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("missing root revision");
|
||||
assert_eq!(stale_baseline.revision, DataUsageCacheRevision::Missing);
|
||||
let mut competing = candidate.clone();
|
||||
competing.scanner_epoch = Some(PROOF_EPOCH + 1);
|
||||
competing.scanner_cycle = Some(PROOF_CYCLE + 1);
|
||||
for state in &mut competing.usage_snapshot_set_states {
|
||||
state.scanner_epoch = Some(PROOF_EPOCH + 1);
|
||||
state.scanner_cycle = Some(PROOF_CYCLE + 1);
|
||||
}
|
||||
let competing_bytes = serde_json::to_vec(&competing).expect("competing root");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), competing_bytes.clone())
|
||||
.await
|
||||
.expect("another publisher wins the actual root slot");
|
||||
let before = read_root(&store).await;
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate, Some(stale_baseline)).await;
|
||||
assert_eq!(
|
||||
publication.outcome(),
|
||||
DataUsagePersistOutcome::Current,
|
||||
"the old missing revision loses CAS and reconciles the newer root"
|
||||
);
|
||||
let (_, _, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty);
|
||||
assert_eq!(
|
||||
read_root(&store).await,
|
||||
before,
|
||||
"bucket durability must not authorize replacing the winning root"
|
||||
);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_cannot_transfer_proof_between_real_scan_results() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (first_scan, first_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let (second_scan, second_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
assert_eq!(first_candidate.scanner_epoch, second_candidate.scanner_epoch);
|
||||
assert_eq!(first_candidate.scanner_cycle, second_candidate.scanner_cycle);
|
||||
assert_eq!(first_candidate.objects_total_count, second_candidate.objects_total_count);
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let publication = publish_candidate(&store, &first_scan, first_candidate, Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
assert!(read_root(&store).await.0.is_some(), "the first scan really published its root");
|
||||
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(second_scan, publication);
|
||||
assert!(pending, "another scan's publication must not finish this scan's dirty maintenance work");
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"same counters and cycle cannot transfer another scan's proof"
|
||||
);
|
||||
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_stale_baseline_cannot_prove_a_replaced_root() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (first_scan, first_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&first_candidate).expect("first candidate"),
|
||||
)
|
||||
.await
|
||||
.expect("persist the first candidate on real disks");
|
||||
let stale_baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("capture the genuine first root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let mut reader = PutObjReader::from_vec(b"second".to_vec());
|
||||
store.pools[0].disk_set[0]
|
||||
.put_object(PROOF_BUCKET, "second", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("commit a real namespace change");
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"direct storage writes leave this fixture's scanner hint generation unchanged"
|
||||
);
|
||||
let (_, replacement) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
assert_eq!(first_candidate.scanner_epoch, replacement.scanner_epoch);
|
||||
assert_eq!(first_candidate.scanner_cycle, replacement.scanner_cycle);
|
||||
assert_eq!((first_candidate.objects_total_count, replacement.objects_total_count), (1, 2));
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&replacement).expect("replacement candidate"),
|
||||
)
|
||||
.await
|
||||
.expect("publish the replacement root");
|
||||
let current = read_root(&store).await;
|
||||
assert_ne!(current.1, stale_baseline.revision);
|
||||
|
||||
// The supplied baseline still equals candidate A, but the actual target
|
||||
// now contains B. Compatibility's AlreadyDurable outcome is not proof.
|
||||
let publication = publish_candidate(&store, &first_scan, first_candidate, Some(stale_baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(first_scan, publication);
|
||||
assert!(pending);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty);
|
||||
assert_eq!(read_root(&store).await, current);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish() {
|
||||
for mutation in ["remote_ack_target", "publication_epoch", "remote_lease_targets"] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let changed_generation = dirty
|
||||
.get(PROOF_BUCKET)
|
||||
.expect("the real scan has dirty work")
|
||||
.checked_add(1)
|
||||
.expect("bounded fixture generation");
|
||||
let changed_epoch = scan
|
||||
.publication_epoch()
|
||||
.expect("real scan publication epoch")
|
||||
.checked_add(1)
|
||||
.expect("bounded fixture epoch");
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved, "{mutation}");
|
||||
let root_before = read_root(&store).await;
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(root_before.0.as_deref().expect("actual saved root"))
|
||||
.expect("persisted root JSON"),
|
||||
candidate,
|
||||
"{mutation}: the original candidate really reached root storage"
|
||||
);
|
||||
|
||||
let changed = match mutation {
|
||||
"remote_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "proof-peer:9000".to_string(),
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
|
||||
}]),
|
||||
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
|
||||
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
|
||||
"proof-peer:9000".to_string(),
|
||||
crate::scanner_activity_epoch().to_string(),
|
||||
changed_generation,
|
||||
)]),
|
||||
_ => unreachable!("fixed mutation cases"),
|
||||
};
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(changed, publication);
|
||||
assert!(
|
||||
acknowledgements.is_empty(),
|
||||
"{mutation}: the old root proof must not authorize changed ACK work"
|
||||
);
|
||||
assert!(pending, "{mutation}: changed maintenance work must remain pending");
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"{mutation}: the changed scan must not clear local dirty work"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, root_before, "{mutation}: the durable original root is retained");
|
||||
}
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
@@ -34,6 +34,139 @@ pub(super) enum DataUsagePersistOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RootPublicationProof {
|
||||
candidate: crate::scanner_io::ScannerPublicationExpectation,
|
||||
root_version: (String, [u8; 32]),
|
||||
}
|
||||
|
||||
impl RootPublicationProof {
|
||||
pub(crate) fn verified_version_for(
|
||||
&self,
|
||||
expected: &crate::scanner_io::ScannerPublicationExpectation,
|
||||
) -> Option<&(String, [u8; 32])> {
|
||||
self.candidate.same_candidate(expected).then_some(&self.root_version)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct DataUsagePublicationResult {
|
||||
outcome: DataUsagePersistOutcome,
|
||||
proof: Option<RootPublicationProof>,
|
||||
}
|
||||
|
||||
impl From<DataUsagePersistOutcome> for DataUsagePublicationResult {
|
||||
fn from(outcome: DataUsagePersistOutcome) -> Self {
|
||||
Self { outcome, proof: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsagePublicationResult {
|
||||
pub(super) fn outcome(&self) -> DataUsagePersistOutcome {
|
||||
self.outcome
|
||||
}
|
||||
pub(super) fn restrict_outcome(&mut self, outcome: DataUsagePersistOutcome) {
|
||||
if outcome != self.outcome {
|
||||
self.proof = None;
|
||||
}
|
||||
self.outcome = outcome;
|
||||
}
|
||||
pub(super) fn into_parts(self) -> (DataUsagePersistOutcome, Option<RootPublicationProof>) {
|
||||
(self.outcome, self.proof)
|
||||
}
|
||||
}
|
||||
|
||||
fn root_ack_write_is_confirmed<T, E>(
|
||||
result: &std::result::Result<T, E>,
|
||||
state: Option<ScannerPublicationCommitState>,
|
||||
written_etag: Option<&str>,
|
||||
) -> bool {
|
||||
result.is_ok() && state == Some(ScannerPublicationCommitState::Committed) && written_etag.is_some_and(|etag| !etag.is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod root_publication_confirmation_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn root_publication_confirmation_requires_committed_state_and_write_revision() {
|
||||
let saved = Ok::<(), ()>(());
|
||||
for state in [
|
||||
None,
|
||||
Some(ScannerPublicationCommitState::Admitted),
|
||||
Some(ScannerPublicationCommitState::InFlight),
|
||||
Some(ScannerPublicationCommitState::AbortedBeforeCommit),
|
||||
Some(ScannerPublicationCommitState::Indeterminate),
|
||||
] {
|
||||
assert!(!root_ack_write_is_confirmed(&saved, state, Some("revision")));
|
||||
}
|
||||
for etag in [None, Some("")] {
|
||||
assert!(!root_ack_write_is_confirmed(&saved, Some(ScannerPublicationCommitState::Committed), etag));
|
||||
}
|
||||
assert!(root_ack_write_is_confirmed(
|
||||
&saved,
|
||||
Some(ScannerPublicationCommitState::Committed),
|
||||
Some("revision")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_publication_confirmation_does_not_carry_state_across_cas_attempts() {
|
||||
let attempts = [
|
||||
(Err(()), Some(ScannerPublicationCommitState::Committed), Some("first")),
|
||||
(Ok(()), Some(ScannerPublicationCommitState::AbortedBeforeCommit), Some("second")),
|
||||
(Ok(()), None, Some("legacy")),
|
||||
(Ok(()), Some(ScannerPublicationCommitState::Committed), Some("confirmed")),
|
||||
];
|
||||
let confirmations = attempts
|
||||
.iter()
|
||||
.map(|(result, state, etag)| root_ack_write_is_confirmed(result, *state, *etag))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(confirmations, [false, false, false, true]);
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_root_publication_proof<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
ctx: &CancellationToken,
|
||||
deadline: tokio::time::Instant,
|
||||
epoch: u64,
|
||||
expected: &crate::scanner_io::ScannerPublicationExpectation,
|
||||
candidate: &DataUsageInfo,
|
||||
written_etag: Option<&str>,
|
||||
) -> Option<RootPublicationProof> {
|
||||
let read = async {
|
||||
let _admission = scanner_publication_admission_for_epoch(store.clone(), epoch).await?;
|
||||
let (bytes, revision) = read_config_with_revision(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.ok()?;
|
||||
let bytes = bytes?;
|
||||
let DataUsageCacheRevision::Etag(etag) = revision else {
|
||||
return None;
|
||||
};
|
||||
if etag.is_empty() || written_etag.is_some_and(|written| written != etag) {
|
||||
return None;
|
||||
}
|
||||
let persisted: DataUsageInfo = serde_json::from_slice(&bytes).ok()?;
|
||||
if &persisted != candidate {
|
||||
return None;
|
||||
}
|
||||
let root_digest = Sha256::digest(&bytes).into();
|
||||
if ctx.is_cancelled() || tokio::time::Instant::now() >= deadline {
|
||||
return None;
|
||||
}
|
||||
Some(RootPublicationProof {
|
||||
candidate: expected.clone(),
|
||||
root_version: (etag, root_digest),
|
||||
})
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => None,
|
||||
result = tokio::time::timeout_at(deadline, read) => result.ok().flatten(),
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
|
||||
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
|
||||
}
|
||||
@@ -166,6 +299,7 @@ pub(super) struct ScannerPublicationFence {
|
||||
pub(super) scanner_publication_lease_fence: Option<String>,
|
||||
pub(super) remote_lease_tokens: Vec<Uuid>,
|
||||
pub(super) lease_release_safe: Arc<AtomicBool>,
|
||||
pub(super) ack_expectation: Option<crate::scanner_io::ScannerPublicationExpectation>,
|
||||
}
|
||||
|
||||
impl ScannerPublicationFence {
|
||||
@@ -180,6 +314,7 @@ impl ScannerPublicationFence {
|
||||
scanner_publication_lease_fence,
|
||||
remote_lease_tokens: Vec::new(),
|
||||
lease_release_safe: Arc::new(AtomicBool::new(true)),
|
||||
ack_expectation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,21 +327,26 @@ impl ScannerPublicationFence {
|
||||
self.lease_release_safe = lease_release_safe;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn with_ack_expectation(mut self, expected: Option<crate::scanner_io::ScannerPublicationExpectation>) -> Self {
|
||||
self.ack_expectation = expected;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum DataUsagePersistTaskResult {
|
||||
Completed(DataUsagePersistOutcome),
|
||||
pub(super) enum DataUsagePersistTaskResult<T = DataUsagePersistOutcome> {
|
||||
Completed(T),
|
||||
Cancelled,
|
||||
TimedOut,
|
||||
JoinFailed(tokio::task::JoinError),
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_data_usage_persist_task(
|
||||
pub(super) async fn wait_for_data_usage_persist_task<T>(
|
||||
ctx: &CancellationToken,
|
||||
task: &mut AbortOnDropHandle<DataUsagePersistOutcome>,
|
||||
task: &mut AbortOnDropHandle<T>,
|
||||
timeout: Duration,
|
||||
) -> DataUsagePersistTaskResult {
|
||||
) -> DataUsagePersistTaskResult<T> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut *task => match result {
|
||||
@@ -320,6 +460,7 @@ where
|
||||
route_probe,
|
||||
)
|
||||
.await
|
||||
.outcome()
|
||||
}
|
||||
|
||||
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence<
|
||||
@@ -333,7 +474,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
initial_baseline: Option<DataUsagePersistBaseline>,
|
||||
publication_fence: ScannerPublicationFence,
|
||||
route_probe: F,
|
||||
) -> DataUsagePersistOutcome
|
||||
) -> DataUsagePublicationResult
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
|
||||
@@ -344,11 +485,15 @@ where
|
||||
scanner_publication_lease_fence,
|
||||
remote_lease_tokens,
|
||||
lease_release_safe,
|
||||
ack_expectation,
|
||||
} = publication_fence;
|
||||
let ack_deadline = scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline);
|
||||
let mut outcome = DataUsagePersistOutcome::NoUpdate;
|
||||
let mut proof = None;
|
||||
let mut next_baseline = initial_baseline;
|
||||
|
||||
'updates: while let Some(mut data_usage_info) = receiver.recv().await {
|
||||
proof = None;
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
if ctx.is_cancelled() {
|
||||
break;
|
||||
@@ -523,10 +668,14 @@ where
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let sha256hex = (!data.is_empty()).then(|| hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower));
|
||||
let data_digest: [u8; 32] = Sha256::digest(&data).into();
|
||||
let sha256hex = (!data.is_empty()).then(|| hex_simd::encode_to_string(data_digest, hex_simd::AsciiCase::Lower));
|
||||
let data = Bytes::from(data);
|
||||
let backup_due = !observational && data_usage_backup_due(&data_usage_info);
|
||||
let mut cas_retry = 0usize;
|
||||
let mut ack_epoch = None;
|
||||
let mut write_confirmed = false;
|
||||
let mut written_etag = None;
|
||||
let save_outcome = loop {
|
||||
if ctx.is_cancelled() {
|
||||
break 'updates;
|
||||
@@ -557,6 +706,7 @@ where
|
||||
} else {
|
||||
None
|
||||
};
|
||||
ack_epoch = Some(publication_epoch_for_save);
|
||||
let (existing_data, revision) = match baseline {
|
||||
Some(baseline) => (baseline.data, baseline.revision),
|
||||
None => match read_config_with_revision(storeapi.clone(), target_path).await {
|
||||
@@ -645,7 +795,7 @@ where
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
let save_result = {
|
||||
let (save_result, commit_state) = {
|
||||
let publication_scope = storeapi
|
||||
.scanner_data_usage_publication_commit_scope_with_release_flag(
|
||||
publication_epoch_for_save,
|
||||
@@ -681,24 +831,33 @@ where
|
||||
.await;
|
||||
drop(legacy_publication_admission);
|
||||
if let Some(scope) = publication_scope {
|
||||
match scope.wait_for_completion().await {
|
||||
ScannerPublicationCommitState::Committed | ScannerPublicationCommitState::AbortedBeforeCommit => {
|
||||
save_result
|
||||
}
|
||||
let state = scope.wait_for_completion().await;
|
||||
let result = match state {
|
||||
ScannerPublicationCommitState::Committed => save_result,
|
||||
ScannerPublicationCommitState::AbortedBeforeCommit => save_result,
|
||||
ScannerPublicationCommitState::Indeterminate
|
||||
| ScannerPublicationCommitState::Admitted
|
||||
| ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
|
||||
"scanner publication commit scope did not reach a safe terminal state",
|
||||
)),
|
||||
}
|
||||
};
|
||||
(result, Some(state))
|
||||
} else {
|
||||
save_result
|
||||
(save_result, None)
|
||||
}
|
||||
};
|
||||
done_save();
|
||||
|
||||
let attempt_confirmed = root_ack_write_is_confirmed(
|
||||
&save_result,
|
||||
commit_state,
|
||||
save_result.as_ref().ok().and_then(|info| info.etag.as_deref()),
|
||||
);
|
||||
|
||||
match save_result {
|
||||
Ok(object_info) => {
|
||||
write_confirmed = attempt_confirmed;
|
||||
written_etag = object_info.etag.as_ref().filter(|etag| !etag.is_empty()).cloned();
|
||||
if !observational {
|
||||
next_baseline = object_info
|
||||
.etag
|
||||
@@ -909,9 +1068,27 @@ where
|
||||
break 'updates;
|
||||
}
|
||||
}
|
||||
if !observational
|
||||
&& data_usage_info.usage_snapshot_converged == Some(true)
|
||||
&& matches!(outcome, DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable)
|
||||
&& (outcome == DataUsagePersistOutcome::AlreadyDurable || write_confirmed)
|
||||
&& let (Some(expected), Some(epoch)) = (ack_expectation.as_ref(), ack_epoch)
|
||||
&& expected.matches_encoded_candidate(&data_digest)
|
||||
{
|
||||
proof = read_root_publication_proof(
|
||||
storeapi.clone(),
|
||||
&ctx,
|
||||
ack_deadline,
|
||||
epoch,
|
||||
expected,
|
||||
&data_usage_info,
|
||||
written_etag.as_deref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
outcome
|
||||
DataUsagePublicationResult { outcome, proof }
|
||||
}
|
||||
|
||||
async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
|
||||
|
||||
@@ -675,6 +675,28 @@ fn partial_cache_is_useful(root: &DataUsageEntry, pending_heals_changed: bool) -
|
||||
data_usage_root_has_progress(root) || pending_heals_changed
|
||||
}
|
||||
|
||||
/// Process-local hint that narrows a dirty bucket scan to known changed direct children.
|
||||
///
|
||||
/// The hint is used only while rebuilding a complete bucket cache. It never
|
||||
/// changes usage publication semantics and callers must discard it when the
|
||||
/// mutation source cannot be verified.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerBucketPrefixScanScope {
|
||||
selected_top_level_entries: Arc<HashSet<String>>,
|
||||
}
|
||||
|
||||
impl ScannerBucketPrefixScanScope {
|
||||
pub(crate) fn from_dirty_top_level_entries(entries: HashSet<String>) -> Option<Self> {
|
||||
(!entries.is_empty()).then(|| Self {
|
||||
selected_top_level_entries: Arc::new(entries),
|
||||
})
|
||||
}
|
||||
|
||||
fn contains(&self, entry: &str) -> bool {
|
||||
self.selected_top_level_entries.contains(entry)
|
||||
}
|
||||
}
|
||||
|
||||
/// Folder scanner for scanning directory structures
|
||||
pub struct FolderScanner {
|
||||
root: String,
|
||||
@@ -686,6 +708,7 @@ pub struct FolderScanner {
|
||||
heal_object_select: u32,
|
||||
scan_mode: HealScanMode,
|
||||
is_erasure_mode: bool,
|
||||
prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
|
||||
failed_object_ttl_secs: u64,
|
||||
failed_objects_max: usize,
|
||||
@@ -710,6 +733,10 @@ pub struct FolderScanner {
|
||||
coverage_frontier: Option<String>,
|
||||
resume_frontier: Option<String>,
|
||||
coverage_gap: bool,
|
||||
pending_heal_sync_deferred: bool,
|
||||
pending_heal_batch_dirty: bool,
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: usize,
|
||||
pending_size_reconciliation_keys: HashSet<String>,
|
||||
pending_size_reconciliation_scopes: HashSet<String>,
|
||||
pending_size_reconciliation_truncated: bool,
|
||||
@@ -777,6 +804,47 @@ impl FolderScanner {
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn should_reuse_clean_root_child(
|
||||
&self,
|
||||
folder: &CachedFolder,
|
||||
into: &DataUsageEntry,
|
||||
child: &CachedFolder,
|
||||
child_hash: &DataUsageHash,
|
||||
abandoned_children: &DataUsageHashMap,
|
||||
) -> bool {
|
||||
let Some(prefix_scan_scope) = &self.prefix_scan_scope else {
|
||||
return false;
|
||||
};
|
||||
// Erasure-mode usage scans also perform probabilistic object-health
|
||||
// work. Reusing a clean subtree here would silently suppress that
|
||||
// independent maintenance path, so prefix reuse is limited to the
|
||||
// non-erasure data-usage scanner.
|
||||
if self.is_erasure_mode
|
||||
|| folder.parent.is_some()
|
||||
|| folder.name != self.old_cache.info.name
|
||||
|| into.compacted
|
||||
|| !abandoned_children.contains(&child_hash.key())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entry) = child
|
||||
.name
|
||||
.strip_prefix(folder.name.as_str())
|
||||
.and_then(|entry| entry.strip_prefix('/'))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
|
||||
}
|
||||
|
||||
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
|
||||
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
into.add_child(child_hash);
|
||||
}
|
||||
|
||||
fn should_skip_failed(&self, path: &str) -> bool {
|
||||
let ttl = self.failed_object_ttl_secs;
|
||||
if ttl == 0 {
|
||||
@@ -1080,7 +1148,7 @@ impl FolderScanner {
|
||||
scan_mode,
|
||||
result,
|
||||
);
|
||||
if result.is_admitted() {
|
||||
if result.is_admitted() || matches!(priority, HealChannelPriority::Low) {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
@@ -1447,13 +1515,16 @@ impl FolderScanner {
|
||||
continue;
|
||||
}
|
||||
|
||||
abandoned_children.remove(&h.key());
|
||||
|
||||
if exists {
|
||||
if exists && self.should_reuse_clean_root_child(&folder, into, &this, &h, &abandoned_children) {
|
||||
abandoned_children.remove(&h.key());
|
||||
self.reuse_clean_root_child(&this, &h, into);
|
||||
} else if exists {
|
||||
abandoned_children.remove(&h.key());
|
||||
existing_folders.push(this);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
} else {
|
||||
abandoned_children.remove(&h.key());
|
||||
new_folders.push(this);
|
||||
}
|
||||
continue;
|
||||
@@ -1629,11 +1700,15 @@ impl FolderScanner {
|
||||
if !found_object_metadata && !found_erasure_data_directory {
|
||||
for (candidate, exists, _) in erasure_data_directory_candidates {
|
||||
let h = hash_path(&candidate.name);
|
||||
abandoned_children.remove(&h.key());
|
||||
if exists {
|
||||
if exists && self.should_reuse_clean_root_child(&folder, into, &candidate, &h, &abandoned_children) {
|
||||
abandoned_children.remove(&h.key());
|
||||
self.reuse_clean_root_child(&candidate, &h, into);
|
||||
} else if exists {
|
||||
abandoned_children.remove(&h.key());
|
||||
self.update_cache.copy_with_children(&self.old_cache, &h, &candidate.parent);
|
||||
existing_folders.push(candidate);
|
||||
} else {
|
||||
abandoned_children.remove(&h.key());
|
||||
new_folders.push(candidate);
|
||||
}
|
||||
}
|
||||
@@ -1828,7 +1903,12 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
FolderScanSource::Existing => {
|
||||
if !forward_sweep && !into.compacted && self.old_cache.is_compacted(&h) {
|
||||
// Usage sampling is not proof that a Deep check ran.
|
||||
if self.scan_mode != HealScanMode::Deep
|
||||
&& !forward_sweep
|
||||
&& !into.compacted
|
||||
&& self.old_cache.is_compacted(&h)
|
||||
{
|
||||
let next_cycle = self.old_cache.info.next_cycle as u32;
|
||||
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
|
||||
// Transfer and add as child...
|
||||
@@ -2368,6 +2448,21 @@ pub async fn scan_data_folder(
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
scan_data_folder_scoped(ctx, budget, disks, local_disk, cache, updates, scan_mode, sleeper, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn scan_data_folder_scoped(
|
||||
ctx: CancellationToken,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
disks: Vec<Arc<Disk>>,
|
||||
local_disk: Arc<Disk>,
|
||||
cache: DataUsageCache,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
use crate::data_usage_define::DATA_USAGE_ROOT;
|
||||
|
||||
@@ -2419,6 +2514,7 @@ pub async fn scan_data_folder(
|
||||
heal_object_select,
|
||||
scan_mode,
|
||||
is_erasure_mode,
|
||||
prefix_scan_scope,
|
||||
failed_object_ttl_secs: failed_object_ttl,
|
||||
failed_objects_max,
|
||||
sleeper,
|
||||
@@ -2435,6 +2531,10 @@ pub async fn scan_data_folder(
|
||||
coverage_frontier: resume_frontier.clone(),
|
||||
resume_frontier,
|
||||
coverage_gap: false,
|
||||
pending_heal_sync_deferred: false,
|
||||
pending_heal_batch_dirty: false,
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: 0,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
|
||||
@@ -11,13 +11,58 @@
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The pending-scanner-heal ledger: durable heal intents recorded during scans and retried after MRF consumption.
|
||||
/// Persisted best-effort retry hints. Existing TTL/count pruning applies only
|
||||
/// to this hint cache, never to a committed durable repair obligation.
|
||||
use super::*;
|
||||
|
||||
const PENDING_HEAL_RETRY_BASE_SECS: u64 = 15 * 60;
|
||||
const PENDING_HEAL_RETRY_CAP_SECS: u64 = 6 * 60 * 60;
|
||||
|
||||
pub(super) struct PendingHealSyncBatch<'a> {
|
||||
pub(super) scanner: &'a mut FolderScanner,
|
||||
}
|
||||
|
||||
impl<'a> PendingHealSyncBatch<'a> {
|
||||
pub(super) fn new(scanner: &'a mut FolderScanner) -> Self {
|
||||
scanner.pending_heal_sync_deferred = true;
|
||||
scanner.pending_heal_batch_dirty = false;
|
||||
Self { scanner }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingHealSyncBatch<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.scanner.pending_heal_sync_deferred = false;
|
||||
if self.scanner.pending_heal_batch_dirty {
|
||||
self.scanner.pending_heal_batch_dirty = false;
|
||||
self.scanner.sync_pending_heals();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_pending_heal_attempt(entry: &mut PendingScannerHeal, now: u64) {
|
||||
entry.last_attempt = now;
|
||||
entry.attempts = entry.attempts.saturating_add(1);
|
||||
}
|
||||
|
||||
pub(super) fn observe_pending_heal_admission(entry: &mut PendingScannerHeal, result: HealAdmissionResult) {
|
||||
// Rediscovery and coalesced admissions must not postpone an armed retry.
|
||||
entry.last_admission_result = result.result_label().to_string();
|
||||
entry.last_admission_reason = result.reason_label().to_string();
|
||||
}
|
||||
|
||||
impl FolderScanner {
|
||||
pub(super) fn sync_pending_heals(&mut self) {
|
||||
self.update_cache.info.pending_heals = self.new_cache.info.pending_heals.clone();
|
||||
self.pending_heals_changed = true;
|
||||
if self.pending_heal_sync_deferred {
|
||||
self.pending_heal_batch_dirty = true;
|
||||
return;
|
||||
}
|
||||
self.update_cache.info.pending_heals = self.new_cache.info.pending_heals.clone();
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.pending_heal_sync_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn clear_pending_scanner_heal(
|
||||
@@ -37,32 +82,6 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Batched variant of [`Self::clear_pending_scanner_heal`] for repaired
|
||||
/// notices (backlog#1894 axis B): one retain pass and one ledger sync
|
||||
/// for the whole notice set, so a mass-recovery first sweep cannot turn
|
||||
/// into thousands of full-table clones on the scan task. Only Object
|
||||
/// entries match — bucket-level heals are never the MRF consumer's work.
|
||||
pub(super) fn clear_pending_scanner_heals_for_repaired(&mut self, events: &[rustfs_common::mrf_channel::MrfRepairedEvent]) {
|
||||
// Pre-resolve the notice version strings once; each ledger entry then
|
||||
// compares against plain Option<&str>.
|
||||
let targets: Vec<(&str, &str, Option<String>)> = events
|
||||
.iter()
|
||||
.map(|event| (event.bucket.as_ref(), event.object.as_ref(), mrf_repaired_version_id(event.version_id)))
|
||||
.collect();
|
||||
let before = self.new_cache.info.pending_heals.len();
|
||||
self.new_cache.info.pending_heals.retain(|entry| {
|
||||
entry.kind != PendingScannerHealKind::Object
|
||||
|| !targets.iter().any(|(bucket, object, version)| {
|
||||
entry.bucket.as_str() == *bucket
|
||||
&& entry.object.as_deref() == Some(*object)
|
||||
&& entry.version_id.as_deref() == version.as_deref()
|
||||
})
|
||||
});
|
||||
if self.new_cache.info.pending_heals.len() != before {
|
||||
self.sync_pending_heals();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_pending_scanner_heal(
|
||||
&mut self,
|
||||
kind: PendingScannerHealKind,
|
||||
@@ -80,10 +99,7 @@ impl FolderScanner {
|
||||
.iter_mut()
|
||||
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
|
||||
{
|
||||
entry.last_attempt = now;
|
||||
entry.attempts = entry.attempts.saturating_add(1);
|
||||
entry.last_admission_result = result.result_label().to_string();
|
||||
entry.last_admission_reason = result.reason_label().to_string();
|
||||
observe_pending_heal_admission(entry, result);
|
||||
self.sync_pending_heals();
|
||||
return;
|
||||
}
|
||||
@@ -198,47 +214,54 @@ impl FolderScanner {
|
||||
result: HealAdmissionResult,
|
||||
) {
|
||||
match result {
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {
|
||||
self.clear_pending_scanner_heal(kind, bucket, object, version_id);
|
||||
}
|
||||
HealAdmissionResult::Full | HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull) => {
|
||||
self.record_pending_scanner_heal(kind, bucket, object, version_id, scan_mode, result);
|
||||
}
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped) => {
|
||||
self.clear_pending_scanner_heal(kind, bucket, object, version_id);
|
||||
}
|
||||
// Admin-only overlap rejections (HS-06); the scanner never sees
|
||||
// them, but if it ever does, treat them as terminal like any
|
||||
// other policy drop rather than endlessly retrying.
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
HealAdmissionResult::Accepted
|
||||
| HealAdmissionResult::Merged
|
||||
| HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
|
||||
| HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
| HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths) => {
|
||||
self.clear_pending_scanner_heal(kind, bucket, object, version_id);
|
||||
// Admission is neither repair completion nor a durable
|
||||
// successor receipt. Preserve existing responsibility without
|
||||
// turning every newly admitted hint into a persisted intent.
|
||||
if let Some(entry) = self
|
||||
.new_cache
|
||||
.info
|
||||
.pending_heals
|
||||
.iter_mut()
|
||||
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
|
||||
{
|
||||
observe_pending_heal_admission(entry, result);
|
||||
self.sync_pending_heals();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn retry_pending_scanner_heals(&mut self) -> Result<(), ScannerError> {
|
||||
if !self.should_heal().await {
|
||||
let batch = PendingHealSyncBatch::new(self);
|
||||
let scanner = &mut *batch.scanner;
|
||||
if !scanner.should_heal().await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bucket = self.new_cache.info.name.clone();
|
||||
// Backlog#1894 axis B: repairs the MRF consumer landed hand the
|
||||
// manager the heal task, so the matching pending-ledger entries are
|
||||
// retried nowhere — drop them here. Best-effort: a lost notice just
|
||||
// leaves the entry to expire through its own attempts/age limits.
|
||||
let bucket = scanner.new_cache.info.name.clone();
|
||||
// Legacy notices cannot bind a verified disposition to the current
|
||||
// incarnation, kind, set scope and responsibility generation.
|
||||
let repaired = rustfs_common::mrf_channel::take_mrf_repaired_events_for(&bucket);
|
||||
if !repaired.is_empty() {
|
||||
self.clear_pending_scanner_heals_for_repaired(&repaired);
|
||||
counter!("rustfs_scanner_unverified_repair_notices_total")
|
||||
.increment(u64::try_from(repaired.len()).unwrap_or(u64::MAX));
|
||||
}
|
||||
self.prune_pending_scanner_heals();
|
||||
for pending in pending_scanner_heal_retry_candidates(&self.new_cache.info.pending_heals, &bucket) {
|
||||
if !self.should_heal().await {
|
||||
scanner.prune_pending_scanner_heals();
|
||||
for pending in pending_scanner_heal_retry_candidates(&scanner.new_cache.info.pending_heals, &bucket) {
|
||||
if !scanner.should_heal().await {
|
||||
break;
|
||||
}
|
||||
|
||||
let Some(request) = build_pending_scanner_heal_request(&pending) else {
|
||||
self.clear_pending_scanner_heal(pending.kind, &pending.bucket, None, pending.version_id.as_deref());
|
||||
scanner.clear_pending_scanner_heal(pending.kind, &pending.bucket, None, pending.version_id.as_deref());
|
||||
counter!(
|
||||
METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL,
|
||||
"bucket" => pending.bucket.clone(),
|
||||
@@ -257,14 +280,27 @@ impl FolderScanner {
|
||||
continue;
|
||||
};
|
||||
|
||||
self.send_required_scanner_heal_request(
|
||||
pending.kind,
|
||||
pending.bucket.clone(),
|
||||
pending.object.clone(),
|
||||
pending.version_id.clone(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
if let Some(entry) = scanner.new_cache.info.pending_heals.iter_mut().find(|entry| {
|
||||
pending_scanner_heal_matches(
|
||||
entry,
|
||||
pending.kind,
|
||||
&pending.bucket,
|
||||
pending.object.as_deref(),
|
||||
pending.version_id.as_deref(),
|
||||
)
|
||||
}) {
|
||||
record_pending_heal_attempt(entry, Self::now_secs());
|
||||
scanner.sync_pending_heals();
|
||||
}
|
||||
scanner
|
||||
.send_required_scanner_heal_request(
|
||||
pending.kind,
|
||||
pending.bucket.clone(),
|
||||
pending.object.clone(),
|
||||
pending.version_id.clone(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -295,16 +331,6 @@ pub(super) fn pending_scanner_heal_identity(entry: &PendingScannerHeal) -> (u8,
|
||||
(kind, entry.bucket.as_str(), entry.object.as_deref(), entry.version_id.as_deref())
|
||||
}
|
||||
|
||||
/// Decode an MRF repaired-notice version id for ledger matching. A nil UUID
|
||||
/// means "no value" per the repo-wide defensive-UUID invariant, so it maps
|
||||
/// to `None` and matches unversioned ledger entries only.
|
||||
pub(super) fn mrf_repaired_version_id(version_id: Option<[u8; 16]>) -> Option<String> {
|
||||
version_id
|
||||
.map(uuid::Uuid::from_bytes)
|
||||
.filter(|uuid| !uuid.is_nil())
|
||||
.map(|uuid| uuid.to_string())
|
||||
}
|
||||
|
||||
pub(super) fn sort_pending_scanner_heals_for_retry(entries: &mut [PendingScannerHeal]) {
|
||||
entries.sort_by(|a, b| {
|
||||
a.last_attempt
|
||||
@@ -318,30 +344,56 @@ pub(super) fn pending_scanner_heal_retry_candidates(
|
||||
pending_heals: &[PendingScannerHeal],
|
||||
bucket: &str,
|
||||
) -> Vec<PendingScannerHeal> {
|
||||
let mut entries: Vec<PendingScannerHeal> = pending_heals.iter().filter(|entry| entry.bucket == bucket).cloned().collect();
|
||||
sort_pending_scanner_heals_for_retry(&mut entries);
|
||||
pending_scanner_heal_retry_candidates_at(pending_heals, bucket, FolderScanner::now_secs())
|
||||
}
|
||||
|
||||
pub(super) fn pending_scanner_heal_retry_candidates_at(
|
||||
pending_heals: &[PendingScannerHeal],
|
||||
bucket: &str,
|
||||
now: u64,
|
||||
) -> Vec<PendingScannerHeal> {
|
||||
// Schedule across scanner cycles rather than allocating a timer per hint.
|
||||
// A later Full response must not reset an already retried hint's backoff.
|
||||
let mut entries: Vec<&PendingScannerHeal> = pending_heals
|
||||
.iter()
|
||||
.filter(|entry| {
|
||||
let exponent = entry.attempts.saturating_sub(1).min(31);
|
||||
let delay = PENDING_HEAL_RETRY_BASE_SECS
|
||||
.saturating_mul(1_u64 << exponent)
|
||||
.min(PENDING_HEAL_RETRY_CAP_SECS);
|
||||
entry.bucket == bucket && now.checked_sub(entry.last_attempt).is_some_and(|age| age >= delay)
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by(|a, b| {
|
||||
a.last_attempt
|
||||
.cmp(&b.last_attempt)
|
||||
.then_with(|| a.attempts.cmp(&b.attempts))
|
||||
.then_with(|| pending_scanner_heal_identity(a).cmp(&pending_scanner_heal_identity(b)))
|
||||
});
|
||||
entries.truncate(MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET);
|
||||
entries
|
||||
entries.into_iter().cloned().collect()
|
||||
}
|
||||
|
||||
pub(super) fn build_pending_scanner_heal_request(entry: &PendingScannerHeal) -> Option<HealChannelRequest> {
|
||||
let priority = if entry.last_admission_result == "full"
|
||||
|| (entry.last_admission_result == "dropped" && entry.last_admission_reason == "queue_full")
|
||||
{
|
||||
HealChannelPriority::High
|
||||
} else {
|
||||
HealChannelPriority::Low
|
||||
};
|
||||
match entry.kind {
|
||||
PendingScannerHealKind::Bucket => Some(build_bucket_heal_request(entry.bucket.clone(), HealChannelPriority::High)),
|
||||
PendingScannerHealKind::Bucket => Some(build_bucket_heal_request(entry.bucket.clone(), priority)),
|
||||
PendingScannerHealKind::Object => entry.object.as_ref().map(|object| {
|
||||
if entry.version_id.is_none() {
|
||||
build_non_destructive_object_heal_request(
|
||||
entry.bucket.clone(),
|
||||
object.clone(),
|
||||
entry.scan_mode,
|
||||
HealChannelPriority::High,
|
||||
)
|
||||
build_non_destructive_object_heal_request(entry.bucket.clone(), object.clone(), entry.scan_mode, priority)
|
||||
} else {
|
||||
build_object_heal_request(
|
||||
entry.bucket.clone(),
|
||||
object.clone(),
|
||||
entry.version_id.clone(),
|
||||
entry.scan_mode,
|
||||
HealChannelPriority::High,
|
||||
priority,
|
||||
)
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
use crate::SCANNER_SLEEPER;
|
||||
|
||||
use super::*;
|
||||
|
||||
mod mrf_ownership;
|
||||
use crate::storage_api::VersionPurgeStatusType;
|
||||
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
|
||||
@@ -330,6 +332,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
heal_object_select: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
is_erasure_mode: false,
|
||||
prefix_scan_scope: None,
|
||||
failed_object_ttl_secs: u64::MAX,
|
||||
failed_objects_max: usize::MAX,
|
||||
sleeper: SCANNER_SLEEPER.clone(),
|
||||
@@ -350,6 +353,9 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
coverage_frontier: None,
|
||||
resume_frontier: None,
|
||||
coverage_gap: false,
|
||||
pending_heal_sync_deferred: false,
|
||||
pending_heal_batch_dirty: false,
|
||||
pending_heal_sync_count: 0,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -1132,23 +1138,8 @@ fn pending_heal(
|
||||
}
|
||||
}
|
||||
|
||||
/// The nil-UUID branch of the defensive-UUID invariant: a nil version in
|
||||
/// a repaired notice means "no value" and must match unversioned ledger
|
||||
/// entries only.
|
||||
#[test]
|
||||
fn test_mrf_repaired_version_id_maps_nil_to_none() {
|
||||
assert_eq!(mrf_repaired_version_id(None), None);
|
||||
assert_eq!(mrf_repaired_version_id(Some([0u8; 16])), None);
|
||||
let uuid = Uuid::new_v4();
|
||||
assert_eq!(mrf_repaired_version_id(Some(*uuid.as_bytes())), Some(uuid.to_string()));
|
||||
}
|
||||
|
||||
/// Full wiring of backlog#1894 axis B: notes taken for the scanned bucket
|
||||
/// clear exactly the matching Object ledger entries — bucket-level
|
||||
/// entries, other buckets' entries, and version-mismatched entries
|
||||
/// survive; a real (non-nil) version matches only the same version.
|
||||
#[tokio::test]
|
||||
async fn test_mrf_repaired_notices_clear_matching_ledger_entries() {
|
||||
async fn mrf_ownership_legacy_notices_preserve_pending_entries() {
|
||||
use rustfs_common::mrf_channel::note_mrf_repaired;
|
||||
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
@@ -1176,8 +1167,7 @@ async fn test_mrf_repaired_notices_clear_matching_ledger_entries() {
|
||||
|
||||
note_mrf_repaired("bucket", "object-a", None);
|
||||
note_mrf_repaired("bucket", "object-b", Some(*Uuid::parse_str(&version).unwrap().as_bytes()));
|
||||
// A nil-UUID notice for object-c means "no value": it clears the
|
||||
// unversioned entry but must not touch the versioned one.
|
||||
// Neither nil nor a matching version proves incarnation, scope or owner.
|
||||
note_mrf_repaired("bucket", "object-c", Some([0u8; 16]));
|
||||
// A notice for a target the ledger does not track must be a no-op.
|
||||
note_mrf_repaired("bucket", "object-untracked", None);
|
||||
@@ -1194,12 +1184,12 @@ async fn test_mrf_repaired_notices_clear_matching_ledger_entries() {
|
||||
.iter()
|
||||
.map(|entry| (entry.kind, entry.bucket.as_str(), entry.object.as_deref(), entry.version_id.as_deref()))
|
||||
.collect();
|
||||
// Cleared: object-a (no version), object-b (exact version match), and
|
||||
// object-c's unversioned entry (the nil branch matched no-version
|
||||
// only — the versioned object-c entry survives).
|
||||
assert_eq!(
|
||||
survivors,
|
||||
vec![
|
||||
(PendingScannerHealKind::Object, "bucket", Some("object-a"), None),
|
||||
(PendingScannerHealKind::Object, "bucket", Some("object-b"), Some(version.as_str())),
|
||||
(PendingScannerHealKind::Object, "bucket", Some("object-c"), None),
|
||||
(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
@@ -1338,7 +1328,7 @@ async fn test_pending_heal_update_keeps_stale_entry_until_retry_prune() {
|
||||
);
|
||||
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 2);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].object.as_deref(), Some("object"));
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
}
|
||||
@@ -1371,7 +1361,7 @@ async fn test_pending_heal_queue_full_deduplicates_object_entry() {
|
||||
let pending = &scanner.new_cache.info.pending_heals[0];
|
||||
assert_eq!(pending.object.as_deref(), Some("object"));
|
||||
assert_eq!(pending.version_id.as_deref(), Some("version-a"));
|
||||
assert_eq!(pending.attempts, 2);
|
||||
assert_eq!(pending.attempts, 1);
|
||||
assert_eq!(pending.last_admission_result, "dropped");
|
||||
assert_eq!(pending.last_admission_reason, "queue_full");
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
@@ -1379,7 +1369,7 @@ async fn test_pending_heal_queue_full_deduplicates_object_entry() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pending_heal_admitted_results_clear_matching_entry() {
|
||||
async fn mrf_ownership_admission_preserves_existing_pending() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
|
||||
@@ -1400,7 +1390,8 @@ async fn test_pending_heal_admitted_results_clear_matching_entry() {
|
||||
HealAdmissionResult::Accepted,
|
||||
);
|
||||
|
||||
assert!(scanner.new_cache.info.pending_heals.is_empty());
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].last_admission_result, "accepted");
|
||||
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Bucket,
|
||||
@@ -1419,11 +1410,12 @@ async fn test_pending_heal_admitted_results_clear_matching_entry() {
|
||||
HealAdmissionResult::Merged,
|
||||
);
|
||||
|
||||
assert!(scanner.new_cache.info.pending_heals.is_empty());
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 2);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[1].last_admission_result, "merged");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pending_heal_policy_dropped_clears_without_creating_entry() {
|
||||
async fn mrf_ownership_policy_drop_does_not_discharge_existing_pending() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
|
||||
@@ -1454,7 +1446,7 @@ async fn test_pending_heal_policy_dropped_clears_without_creating_entry() {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped),
|
||||
);
|
||||
|
||||
assert!(scanner.new_cache.info.pending_heals.is_empty());
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2393,6 +2385,115 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
let bucket_dir = temp_dir.join("bucket");
|
||||
tokio::fs::create_dir_all(bucket_dir.join("clean"))
|
||||
.await
|
||||
.expect("failed to create clean top-level directory");
|
||||
tokio::fs::create_dir_all(bucket_dir.join("dirty"))
|
||||
.await
|
||||
.expect("failed to create dirty top-level directory");
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/clean",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 17,
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.old_cache.replace(
|
||||
"bucket/dirty",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 23,
|
||||
objects: 4,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
.await
|
||||
.expect("scoped root scan should finish successfully");
|
||||
|
||||
let clean = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket/clean")
|
||||
.expect("clean entry should be copied from the complete cache");
|
||||
assert_eq!((clean.size, clean.objects), (17, 3));
|
||||
let dirty = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket/dirty")
|
||||
.expect("dirty entry should be rescanned");
|
||||
assert_eq!((dirty.size, dirty.objects), (0, 0));
|
||||
let bucket = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket")
|
||||
.expect("bucket root should include reused and rescanned entries");
|
||||
assert_eq!((bucket.size, bucket.objects), (17, 3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_root_scan_preserves_erasure_health_walks() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
let bucket_dir = temp_dir.join("bucket");
|
||||
tokio::fs::create_dir_all(bucket_dir.join("clean"))
|
||||
.await
|
||||
.expect("failed to create clean top-level directory");
|
||||
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/clean",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 17,
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
.await
|
||||
.expect("erasure root scan should finish successfully");
|
||||
|
||||
let clean = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket/clean")
|
||||
.expect("erasure scan should visit the clean entry");
|
||||
assert_eq!((clean.size, clean.objects), (0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
||||
|
||||
@@ -14,12 +14,13 @@
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
|
||||
use crate::scanner_io::{ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk};
|
||||
use crate::storage_api::scanner_io::ObjectIO;
|
||||
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
mod deep_compacted;
|
||||
mod segment_observation;
|
||||
|
||||
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
|
||||
@@ -28,6 +29,13 @@ const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
|
||||
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
|
||||
|
||||
fn scan_options(scan_mode: HealScanMode) -> ScannerDiskScanOptions {
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
|
||||
#[derive(Debug)]
|
||||
struct FixtureStore {
|
||||
@@ -537,7 +545,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("scan must revisit the prefix");
|
||||
@@ -593,7 +601,7 @@ async fn checkpoint_fixture_failed_child_prevents_receipt_advancing_past_gap() {
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("scan with a known failed child");
|
||||
@@ -644,7 +652,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("initial complete baseline");
|
||||
@@ -689,7 +697,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("sampling interruption");
|
||||
@@ -736,7 +744,14 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(budget.token(), budget, vec![scanner.local_disk.clone()], loaded, None, resume_mode)
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded,
|
||||
None,
|
||||
scan_options(resume_mode),
|
||||
)
|
||||
.await
|
||||
.expect("bounded recovery scan");
|
||||
let (cache, complete) = match result {
|
||||
@@ -852,7 +867,10 @@ async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("budgeted local disk scan returns partial cache");
|
||||
@@ -930,7 +948,10 @@ async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
|
||||
@@ -981,7 +1002,7 @@ async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
scan_options(HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("bounded sweep outcome");
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
const PREFIX: &str = "bucket/prefix";
|
||||
const OBJECTS: u64 = 4;
|
||||
|
||||
struct CompactedFixture {
|
||||
disk: Arc<Disk>,
|
||||
root: std::path::PathBuf,
|
||||
cache: DataUsageCache,
|
||||
identity: crate::DataUsageScanIdentity,
|
||||
store: Arc<FixtureStore>,
|
||||
_cleanup: TestGuard,
|
||||
}
|
||||
|
||||
async fn scan(
|
||||
disk: &Arc<Disk>,
|
||||
cache: DataUsageCache,
|
||||
mode: HealScanMode,
|
||||
max_objects: u64,
|
||||
) -> (ScannerDiskScanOutcome, Arc<ScannerCycleBudget>) {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&CancellationToken::new(),
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(max_objects),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let outcome = disk
|
||||
.clone()
|
||||
.nsscanner_disk(budget.token(), budget.clone(), vec![disk.clone()], cache, None, super::scan_options(mode))
|
||||
.await
|
||||
.expect("bounded real disk scan");
|
||||
(outcome, budget)
|
||||
}
|
||||
|
||||
async fn save_reload(store: &Arc<FixtureStore>, cache: &DataUsageCache) -> DataUsageCache {
|
||||
let revisions = DataUsageCache::default()
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("fixture save revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save compacted checkpoint through real codec and revision checks");
|
||||
let loaded = store.strict_load().await;
|
||||
assert_eq!(loaded.info.snapshot_complete, cache.info.snapshot_complete);
|
||||
assert_eq!(loaded.info.scan_checkpoint, cache.info.scan_checkpoint);
|
||||
assert_eq!(loaded.info.scan_identity, cache.info.scan_identity);
|
||||
assert_eq!(loaded.info.scan_plan_digest, cache.info.scan_plan_digest);
|
||||
assert_eq!(
|
||||
loaded.checked_flatten("bucket").expect("reloaded root").size,
|
||||
cache.checked_flatten("bucket").expect("returned root").size
|
||||
);
|
||||
loaded
|
||||
}
|
||||
|
||||
impl CompactedFixture {
|
||||
async fn new(mode: HealScanMode) -> Self {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for index in 0..OBJECTS {
|
||||
write_checkpoint_object(&root, &format!("prefix/{index:04}"), &[(None, 1)]).await;
|
||||
}
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
scan_mode: mode,
|
||||
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
|
||||
..bound_checkpoint().1
|
||||
};
|
||||
let mut cache = DataUsageCache::default();
|
||||
// The first real scan builds coverage; the second same-plan scan takes
|
||||
// the normal compaction path. No synthetic compacted cache is injected.
|
||||
for cycle in [11, 12] {
|
||||
cache.prepare_bucket_checkpoint("bucket", cycle, 7, SOURCE, PLAN, identity);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, mode, OBJECTS + 1).await;
|
||||
let ScannerDiskScanOutcome::Complete(completed) = outcome else {
|
||||
panic!("fixture baseline must complete")
|
||||
};
|
||||
assert_eq!(budget.progress().0, OBJECTS, "baseline must read every metadata object");
|
||||
cache = completed;
|
||||
}
|
||||
let store = FixtureStore::new();
|
||||
cache = save_reload(&store, &cache).await;
|
||||
assert!(cache.find(PREFIX).expect("baseline prefix").compacted);
|
||||
assert_eq!(cache.checked_flatten("bucket").expect("complete baseline").size, 4);
|
||||
assert!(cache.info.scan_progress.is_none());
|
||||
Self {
|
||||
disk: scanner.local_disk,
|
||||
root,
|
||||
cache,
|
||||
identity,
|
||||
store,
|
||||
_cleanup: cleanup,
|
||||
}
|
||||
}
|
||||
|
||||
fn next_cycle(&self, sampled: bool) -> u64 {
|
||||
let first = self.cache.info.next_cycle + 1;
|
||||
(first..first + 16)
|
||||
.find(|cycle| hash_path(PREFIX).mod_(u32::try_from(*cycle).expect("bounded cycle"), 16) == sampled)
|
||||
.expect("one selected cycle and non-selected cycles exist within the fixed rotation")
|
||||
}
|
||||
|
||||
fn prepare(&mut self, cycle: u64) {
|
||||
let state = crate::scanner_io::current_cache_root_or_prepare_with_generation(
|
||||
&mut self.cache,
|
||||
"bucket",
|
||||
SOURCE,
|
||||
cycle,
|
||||
7,
|
||||
PLAN,
|
||||
crate::scanner_io::DataUsageCacheReuseOptions {
|
||||
checkpoint_identity: Some(self.identity),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(matches!(state, crate::scanner_io::DataUsageCacheScanState::Prepared { .. }));
|
||||
assert_eq!(self.cache.info.scan_identity, Some(self.identity));
|
||||
assert_eq!(self.cache.info.scan_plan_digest, Some(PLAN));
|
||||
assert!(
|
||||
self.cache.info.scan_progress.is_none(),
|
||||
"same-strength complete baseline uses the existing tree"
|
||||
);
|
||||
assert!(self.cache.find(PREFIX).expect("prepared prefix").compacted);
|
||||
}
|
||||
|
||||
async fn change_metadata_without_activity_event(&self) {
|
||||
// This models a local metadata change not announced by a segment
|
||||
// producer. Deep traversal must not depend on a usage-clean signal.
|
||||
// Healing is disabled: the oracle proves metadata re-entry, not repair.
|
||||
write_checkpoint_object(&self.root, "prefix/0000", &[(None, 7)]).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn deep_compacted_same_plan_rechecks_unsampled_prefix() {
|
||||
temp_env::async_with_vars([(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"))], async {
|
||||
let mut fixture = CompactedFixture::new(HealScanMode::Deep).await;
|
||||
fixture.change_metadata_without_activity_event().await;
|
||||
fixture.prepare(fixture.next_cycle(false));
|
||||
let (outcome, budget) = scan(&fixture.disk, fixture.cache.clone(), HealScanMode::Deep, OBJECTS + 1).await;
|
||||
assert_eq!(
|
||||
budget.progress().0,
|
||||
OBJECTS,
|
||||
"Deep must inspect compacted children even outside the usage sample cycle"
|
||||
);
|
||||
let ScannerDiskScanOutcome::Complete(cache) = outcome else {
|
||||
panic!("bounded Deep scan must complete")
|
||||
};
|
||||
let loaded = save_reload(&fixture.store, &cache).await;
|
||||
let root = loaded.checked_flatten("bucket").expect("Deep scan root");
|
||||
assert_eq!((root.objects, root.size), (4, 10), "Deep must observe the changed metadata");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn deep_compacted_normal_scan_preserves_periodic_sampling() {
|
||||
temp_env::async_with_vars([(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"))], async {
|
||||
let mut fixture = CompactedFixture::new(HealScanMode::Normal).await;
|
||||
fixture.change_metadata_without_activity_event().await;
|
||||
fixture.prepare(fixture.next_cycle(false));
|
||||
let (outcome, budget) = scan(&fixture.disk, fixture.cache.clone(), HealScanMode::Normal, OBJECTS + 1).await;
|
||||
assert_eq!(budget.progress().0, 0, "Normal retains its existing unsampled-subtree policy");
|
||||
let ScannerDiskScanOutcome::Complete(cache) = outcome else { panic!("normal sampling completes") };
|
||||
assert_eq!(cache.checked_flatten("bucket").expect("sampled root").size, 4);
|
||||
fixture.cache = save_reload(&fixture.store, &cache).await;
|
||||
fixture.prepare(fixture.next_cycle(true));
|
||||
let (outcome, budget) = scan(&fixture.disk, fixture.cache.clone(), HealScanMode::Normal, OBJECTS + 1).await;
|
||||
assert_eq!(budget.progress().0, OBJECTS);
|
||||
let ScannerDiskScanOutcome::Complete(cache) = outcome else {
|
||||
panic!("selected Normal rotation completes")
|
||||
};
|
||||
assert_eq!(cache.checked_flatten("bucket").expect("refreshed root").size, 10);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn deep_compacted_budget_preserves_partial_checkpoint() {
|
||||
temp_env::async_with_vars([(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"))], async {
|
||||
let mut fixture = CompactedFixture::new(HealScanMode::Deep).await;
|
||||
fixture.prepare(fixture.next_cycle(false));
|
||||
let (outcome, budget) = scan(&fixture.disk, fixture.cache.clone(), HealScanMode::Deep, 2).await;
|
||||
assert_eq!(budget.progress().0, 2);
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("Deep must retain budget interruption as partial")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
let loaded = save_reload(&fixture.store, &cache).await;
|
||||
assert!(!loaded.info.snapshot_complete);
|
||||
assert!(loaded.checked_flatten("bucket").expect("partial root").objects > 0);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::EcstoreHealResultItem as HealItem;
|
||||
use crate::storage_api::scanner_io::BucketInfo;
|
||||
use rustfs_common::mrf_channel::{
|
||||
MrfIngressResult, MrfKind, MrfScope, note_mrf_repaired, take_mrf_repaired_events_for, try_send_mrf_intent_typed,
|
||||
};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
mrf_queue::spawn_mrf_consumer,
|
||||
storage::{HealListItem, HealObjectInfo, HealStorageAPI},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_admission_observation_does_not_postpone_retry() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner.new_cache.info.pending_heals.push(pending_heal(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("object"),
|
||||
None,
|
||||
100,
|
||||
2,
|
||||
));
|
||||
for result in [
|
||||
HealAdmissionResult::Accepted,
|
||||
HealAdmissionResult::Merged,
|
||||
HealAdmissionResult::Full,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped),
|
||||
] {
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("object"),
|
||||
None,
|
||||
HealScanMode::Deep,
|
||||
result,
|
||||
);
|
||||
let entry = &scanner.new_cache.info.pending_heals[0];
|
||||
assert_eq!((entry.last_attempt, entry.attempts), (100, 2));
|
||||
assert!(pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", 1899).is_empty());
|
||||
assert_eq!(
|
||||
pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", 1900).len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("new-object"),
|
||||
None,
|
||||
HealScanMode::Deep,
|
||||
HealAdmissionResult::Accepted,
|
||||
);
|
||||
assert_eq!(
|
||||
scanner.new_cache.info.pending_heals.len(),
|
||||
1,
|
||||
"successful admission does not create a new ledger"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_ownership_retry_due_boundaries_and_priority_are_bounded() {
|
||||
let mut entry = pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 100, 1);
|
||||
entry.last_admission_result = "accepted".to_string();
|
||||
assert!(pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 999).is_empty());
|
||||
assert_eq!(
|
||||
pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 1000).len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
build_pending_scanner_heal_request(&entry).expect("request").priority,
|
||||
HealChannelPriority::Low
|
||||
);
|
||||
record_pending_heal_attempt(&mut entry, 1000);
|
||||
observe_pending_heal_admission(&mut entry, HealAdmissionResult::Full);
|
||||
assert!(pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 2799).is_empty());
|
||||
assert_eq!(
|
||||
pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 2800).len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
build_pending_scanner_heal_request(&entry).expect("request").priority,
|
||||
HealChannelPriority::High
|
||||
);
|
||||
entry.attempts = u32::MAX;
|
||||
assert!(pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 22599).is_empty());
|
||||
assert_eq!(
|
||||
pending_scanner_heal_retry_candidates_at(std::slice::from_ref(&entry), "bucket", 22600).len(),
|
||||
1
|
||||
);
|
||||
entry.last_attempt = u64::MAX;
|
||||
assert!(pending_scanner_heal_retry_candidates_at(&[entry], "bucket", 22600).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_full_hint_table_has_bounded_multicycle_work_and_sync() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
let base = 1_700_000_000;
|
||||
scanner.new_cache.info.pending_heals = (0..MAX_PENDING_SCANNER_HEALS_PER_BUCKET)
|
||||
.map(|index| {
|
||||
let mut entry =
|
||||
pending_heal(PendingScannerHealKind::Object, "bucket", Some(&format!("object-{index}")), None, base, 1);
|
||||
entry.first_seen = base;
|
||||
entry.last_admission_result = "accepted".to_string();
|
||||
entry
|
||||
})
|
||||
.collect();
|
||||
let indices: HashMap<String, usize> = scanner
|
||||
.new_cache
|
||||
.info
|
||||
.pending_heals
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, entry)| (entry.object.clone().expect("object identity"), index))
|
||||
.collect();
|
||||
scanner.sync_pending_heals();
|
||||
let initial_syncs = scanner.pending_heal_sync_count;
|
||||
let mut requests = 0usize;
|
||||
let mut nonempty_batches = 0;
|
||||
for minute in 0..24 * 60 {
|
||||
let now = base + minute * 60;
|
||||
let candidates = pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", now);
|
||||
assert!(candidates.len() <= MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET);
|
||||
if candidates.is_empty() {
|
||||
continue;
|
||||
}
|
||||
nonempty_batches += 1;
|
||||
let before = scanner.pending_heal_sync_count;
|
||||
{
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
for candidate in candidates {
|
||||
assert_eq!(
|
||||
build_pending_scanner_heal_request(&candidate)
|
||||
.expect("retry request")
|
||||
.priority,
|
||||
HealChannelPriority::Low
|
||||
);
|
||||
let index = indices[candidate.object.as_ref().expect("object identity")];
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[index], now);
|
||||
observe_pending_heal_admission(
|
||||
&mut batch.scanner.new_cache.info.pending_heals[index],
|
||||
HealAdmissionResult::Accepted,
|
||||
);
|
||||
batch.scanner.sync_pending_heals();
|
||||
requests += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 1, "one table clone per changed retry batch");
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), MAX_PENDING_SCANNER_HEALS_PER_BUCKET);
|
||||
}
|
||||
assert!(requests >= MAX_PENDING_SCANNER_HEALS_PER_BUCKET, "every retained hint receives a retry");
|
||||
assert!(
|
||||
requests <= 7 * MAX_PENDING_SCANNER_HEALS_PER_BUCKET,
|
||||
"15min..6h backoff bounds repeated work within 24h"
|
||||
);
|
||||
assert!(scanner.new_cache.info.pending_heals.iter().all(|entry| entry.attempts >= 2));
|
||||
assert_eq!(scanner.pending_heal_sync_count - initial_syncs, nonempty_batches);
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_cancelled_batch_restores_sync_without_per_item_clones() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner
|
||||
.new_cache
|
||||
.info
|
||||
.pending_heals
|
||||
.push(pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1));
|
||||
let before = scanner.pending_heal_sync_count;
|
||||
let mut work = Box::pin(async {
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[0], 100);
|
||||
batch.scanner.sync_pending_heals();
|
||||
batch.scanner.sync_pending_heals();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
assert!(futures::poll!(&mut work).is_pending());
|
||||
drop(work);
|
||||
assert!(!scanner.pending_heal_sync_deferred);
|
||||
assert!(!scanner.pending_heal_batch_dirty);
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 2);
|
||||
assert!(pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", 101).is_empty());
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
let result: std::result::Result<(), &'static str> = async {
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[0], 200);
|
||||
observe_pending_heal_admission(&mut batch.scanner.new_cache.info.pending_heals[0], HealAdmissionResult::Merged);
|
||||
batch.scanner.sync_pending_heals();
|
||||
Err("injected retry batch failure")
|
||||
}
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 2);
|
||||
assert!(!scanner.pending_heal_sync_deferred);
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct NoticeStorage {
|
||||
calls: std::sync::Mutex<HashMap<String, u32>>,
|
||||
retry_started: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for NoticeStorage {
|
||||
async fn get_object_meta(&self, _: &str, _: &str) -> rustfs_heal::Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> rustfs_heal::Result<Vec<u8>> {
|
||||
Err(rustfs_heal::Error::other("unused decode fixture"))
|
||||
}
|
||||
async fn get_bucket_info(&self, bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
|
||||
Ok(Some(BucketInfo {
|
||||
name: bucket.to_string(),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn object_exists(&self, _: &str, _: &str) -> rustfs_heal::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_: &str,
|
||||
object: &str,
|
||||
_: Option<&str>,
|
||||
_: &HealOpts,
|
||||
) -> rustfs_heal::Result<(HealItem, Option<rustfs_heal::Error>)> {
|
||||
let retry = {
|
||||
let mut calls = self.calls.lock().expect("fixture calls");
|
||||
let count = calls.entry(object.to_string()).or_default();
|
||||
*count += 1;
|
||||
*count > 1
|
||||
};
|
||||
if retry {
|
||||
self.retry_started.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
match object {
|
||||
"grace" => Ok((
|
||||
HealItem::default(),
|
||||
Some(rustfs_heal::Error::Disk(crate::DiskError::other(
|
||||
"dangling object deletion deferred by heal grace window; retry_after_secs=3599; grace_secs=3600",
|
||||
))),
|
||||
)),
|
||||
"failed" => Err(rustfs_heal::Error::other("permanent fixture failure")),
|
||||
"cancelled" => Err(rustfs_heal::Error::TaskCancelled),
|
||||
_ => Ok((HealItem::default(), None)),
|
||||
}
|
||||
}
|
||||
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> rustfs_heal::Result<HealItem> {
|
||||
Ok(HealItem::default())
|
||||
}
|
||||
async fn heal_format(&self, _: bool) -> rustfs_heal::Result<(HealItem, Option<rustfs_heal::Error>)> {
|
||||
Ok((HealItem::default(), None))
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
async fn get_disk_for_resume(&self, _: &str) -> rustfs_heal::Result<crate::DiskStore> {
|
||||
Err(rustfs_heal::Error::other("unused resume fixture"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn mrf_ownership_manager_completion_preserves_scanner_pending() {
|
||||
const CHILD: &str = "RUSTFS_MRF_OWNERSHIP_TEST_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_HEAL_MRF_ENABLE", "true")
|
||||
.output()
|
||||
.expect("isolated ingress test process");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(
|
||||
output.status.success() && stdout.contains("1 passed;"),
|
||||
"{stdout}\n{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
// The production ingress channel is a process singleton; isolation keeps
|
||||
// its receiver and lease generations independent from other scanner tests.
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
let bucket = format!("mrf-ownership-{}", Uuid::new_v4());
|
||||
scanner.new_cache.info.name = bucket.clone();
|
||||
scanner.update_cache.info.name = bucket.clone();
|
||||
scanner.heal_object_select = 1;
|
||||
let storage = Arc::new(NoticeStorage::default());
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
enable_auto_heal: false,
|
||||
mainline_throttle_enable: false,
|
||||
heal_interval: Duration::from_millis(10),
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
manager.start().await.expect("production manager starts");
|
||||
spawn_mrf_consumer(manager.clone());
|
||||
for (index, object) in ["grace", "unknown", "failed", "cancelled"].iter().enumerate() {
|
||||
let version = Uuid::new_v4();
|
||||
scanner.new_cache.info.pending_heals.push(pending_heal(
|
||||
PendingScannerHealKind::Object,
|
||||
&bucket,
|
||||
Some(object),
|
||||
Some(&version.to_string()),
|
||||
1,
|
||||
1,
|
||||
));
|
||||
let scope = Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
});
|
||||
assert_eq!(
|
||||
try_send_mrf_intent_typed(MrfKind::PartialWrite, &bucket, object, Some(version), scope),
|
||||
MrfIngressResult::Enqueued
|
||||
);
|
||||
// Re-admission establishes that the first terminal callback released
|
||||
// its ingress lease. Statistics alone precede notice publication.
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match try_send_mrf_intent_typed(MrfKind::PartialWrite, &bucket, object, Some(version), scope) {
|
||||
MrfIngressResult::Enqueued => break,
|
||||
MrfIngressResult::Coalesced => tokio::task::yield_now().await,
|
||||
other => panic!("unexpected retry ingress result: {other:?}"),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("production terminal releases its ingress lease");
|
||||
tokio::time::timeout(Duration::from_secs(5), storage.retry_started.notified())
|
||||
.await
|
||||
.expect("the real consumer starts the second generation");
|
||||
assert!(
|
||||
take_mrf_repaired_events_for(&bucket).is_empty(),
|
||||
"{object}: task completion must not emit an unproved repair"
|
||||
);
|
||||
if *object == "unknown" {
|
||||
assert_eq!(
|
||||
manager.get_statistics().await.total_objects_healed,
|
||||
1,
|
||||
"legacy healed count is not repair proof"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
try_send_mrf_intent_typed(MrfKind::PartialWrite, &bucket, object, Some(version), scope),
|
||||
MrfIngressResult::Coalesced,
|
||||
"the in-flight retry retains its new ingress lease"
|
||||
);
|
||||
note_mrf_repaired(&bucket, object, Some(*version.as_bytes()));
|
||||
let syncs_before_retry = scanner.pending_heal_sync_count;
|
||||
scanner
|
||||
.retry_pending_scanner_heals()
|
||||
.await
|
||||
.expect("real scanner ledger retry");
|
||||
assert_eq!(scanner.pending_heal_sync_count, syncs_before_retry + 1, "the real retry batch syncs once");
|
||||
assert_eq!(
|
||||
scanner.new_cache.info.pending_heals.len(),
|
||||
index + 1,
|
||||
"{object}: pending responsibility survives"
|
||||
);
|
||||
let restored = DataUsageCache::unmarshal(&scanner.new_cache.marshal_msg().expect("serialize pending cache"))
|
||||
.expect("decode pending cache");
|
||||
assert_eq!(restored.info.pending_heals.len(), index + 1);
|
||||
assert_eq!(
|
||||
manager
|
||||
.cancel_tasks_for_path(&format!("{bucket}/{object}"))
|
||||
.await
|
||||
.expect("cancel blocked retry"),
|
||||
1
|
||||
);
|
||||
}
|
||||
manager.stop().await.expect("production manager stops");
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::data_usage_define::{DATA_USAGE_CACHE_KEY_FORMAT, DataUsageCacheRevisions};
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
||||
use crate::scanner_folder::{ScannerBucketPrefixScanScope, ScannerItem, scan_data_folder_scoped};
|
||||
use crate::sleeper::SCANNER_SLEEPER;
|
||||
use crate::{
|
||||
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome,
|
||||
@@ -27,7 +27,7 @@ use metrics::counter;
|
||||
use rand::seq::SliceRandom as _;
|
||||
#[cfg(test)]
|
||||
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
|
||||
use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
|
||||
use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo, observed_data_usage_is_newer};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use rustfs_lock::{LockError, NamespaceLockGuard};
|
||||
@@ -103,6 +103,7 @@ pub type DirtyUsageBuckets = HashMap<String, u64>;
|
||||
#[derive(Clone, Debug)]
|
||||
struct DirtyUsageSnapshot {
|
||||
buckets: Arc<DirtyUsageBuckets>,
|
||||
scopes: Arc<DirtyUsageBucketScopes>,
|
||||
generation: u64,
|
||||
covers_all_pending: bool,
|
||||
}
|
||||
@@ -110,25 +111,41 @@ struct DirtyUsageSnapshot {
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct ScannerBucketScanScope {
|
||||
selected_buckets: Option<Arc<HashSet<String>>>,
|
||||
selected_bucket_prefixes: Option<Arc<HashMap<String, ScannerBucketPrefixScanScope>>>,
|
||||
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
}
|
||||
|
||||
impl ScannerBucketScanScope {
|
||||
fn is_default(&self) -> bool {
|
||||
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
|
||||
#[cfg(test)]
|
||||
pub(crate) fn selected_buckets_for_tests(&self) -> Option<&HashSet<String>> {
|
||||
self.selected_buckets.as_deref()
|
||||
}
|
||||
|
||||
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
|
||||
fn is_default(&self) -> bool {
|
||||
self.selected_buckets.is_none() && self.selected_bucket_prefixes.is_none() && self.baseline_scan_plan_digest.is_none()
|
||||
}
|
||||
|
||||
fn from_dirty_buckets(
|
||||
selected_buckets: HashSet<String>,
|
||||
selected_bucket_prefixes: HashMap<String, ScannerBucketPrefixScanScope>,
|
||||
baseline_scan_plan_digest: DataUsageScanPlanDigest,
|
||||
) -> Self {
|
||||
Self {
|
||||
selected_buckets: Some(Arc::new(selected_buckets)),
|
||||
selected_bucket_prefixes: (!selected_bucket_prefixes.is_empty()).then(|| Arc::new(selected_bucket_prefixes)),
|
||||
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prefix_scope_for(&self, bucket: &str) -> Option<ScannerBucketPrefixScanScope> {
|
||||
self.selected_bucket_prefixes.as_ref()?.get(bucket).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct ScannerCacheBaselineProof<'a> {
|
||||
pub(super) data: Option<&'a Bytes>,
|
||||
pub(super) authoritative_data: Option<&'a Bytes>,
|
||||
pub(super) observed_candidate_data: Option<&'a Bytes>,
|
||||
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
|
||||
pub(super) leader_epoch: u64,
|
||||
pub(super) want_cycle: u64,
|
||||
@@ -142,19 +159,31 @@ struct ScannerPeerDirtyUsageExpectation {
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage_buckets(
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct VerifiedRemoteDirtyUsage {
|
||||
dirty_buckets: HashSet<String>,
|
||||
acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolutionResult {
|
||||
scope: ScannerBucketScanScope,
|
||||
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage(
|
||||
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
|
||||
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
|
||||
) -> Option<HashSet<String>> {
|
||||
) -> Option<VerifiedRemoteDirtyUsage> {
|
||||
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
|
||||
let mut dirty_buckets = HashSet::new();
|
||||
let mut acknowledgements = Vec::new();
|
||||
for (host, snapshot) in peer_snapshots {
|
||||
let expected = expected_peers.get(&host)?;
|
||||
if !received_peers.insert(host)
|
||||
if !received_peers.insert(host.clone())
|
||||
|| snapshot.instance_id != expected.instance_id
|
||||
|| snapshot.generation != expected.generation
|
||||
|| snapshot.generation == u64::MAX
|
||||
@@ -165,27 +194,118 @@ fn verified_remote_dirty_usage_buckets(
|
||||
{
|
||||
return None;
|
||||
}
|
||||
dirty_buckets.extend(snapshot.buckets.into_keys());
|
||||
let entries = snapshot
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|(bucket, state)| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: bucket.clone(),
|
||||
bucket_incarnation: state.bucket_incarnation,
|
||||
generation: state.generation,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
dirty_buckets.extend(snapshot.buckets.keys().cloned());
|
||||
if !entries.is_empty() {
|
||||
acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id: snapshot.instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: snapshot.owner_id,
|
||||
entries,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
|
||||
(received_peers.len() == expected_peers.len()).then_some(VerifiedRemoteDirtyUsage {
|
||||
dirty_buckets,
|
||||
acknowledgements,
|
||||
})
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
|
||||
let data = proof.data?;
|
||||
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
|
||||
if !baseline.is_complete_bucket_usage_snapshot()
|
||||
|| baseline.usage_snapshot_partial
|
||||
|| baseline.usage_snapshot_converged != Some(true)
|
||||
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
|
||||
fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(
|
||||
acknowledgements: &[crate::scanner::ScannerDirtyUsageAcknowledgement],
|
||||
) -> bool {
|
||||
acknowledgements.iter().any(|acknowledgement| {
|
||||
matches!(
|
||||
&acknowledgement.kind,
|
||||
crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. }
|
||||
if entries.len() > crate::SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_remote_dirty_usage_scope(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
mut dirty_buckets: HashSet<String>,
|
||||
remote_dirty_usage: VerifiedRemoteDirtyUsage,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerBucketScopeResolutionResult {
|
||||
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
};
|
||||
|
||||
dirty_buckets.extend(remote_dirty_usage.dirty_buckets);
|
||||
// Peer snapshots contribute bucket names only; the local prefix scopes
|
||||
// would narrow a bucket a peer dirtied elsewhere, so the merged scope
|
||||
// stays at bucket granularity (same rule as the local fallthrough).
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(requested_scope, dirty_buckets, None, true, all_buckets, baseline_proof);
|
||||
if scope.is_default() {
|
||||
return default_result(scope);
|
||||
}
|
||||
let Some(selected_buckets) = scope.selected_buckets.as_ref() else {
|
||||
return default_result(scope);
|
||||
};
|
||||
let mut scoped_acknowledgements = Vec::with_capacity(remote_dirty_usage.acknowledgements.len());
|
||||
for acknowledgement in remote_dirty_usage.acknowledgements {
|
||||
let crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
|
||||
} = acknowledgement
|
||||
else {
|
||||
return default_result(scope);
|
||||
};
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.filter(|entry| selected_buckets.contains(&entry.bucket))
|
||||
.collect::<Vec<_>>();
|
||||
if !entries.is_empty() {
|
||||
scoped_acknowledgements.push(crate::scanner::ScannerDirtyUsageAcknowledgement {
|
||||
host,
|
||||
instance_id,
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries },
|
||||
});
|
||||
}
|
||||
}
|
||||
if scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&scoped_acknowledgements) {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
|
||||
ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
|
||||
}
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_snapshot_plan_digest(
|
||||
snapshot: &DataUsageInfo,
|
||||
proof: ScannerCacheBaselineProof<'_>,
|
||||
expected_converged: bool,
|
||||
) -> Option<DataUsageScanPlanDigest> {
|
||||
if !snapshot.is_complete_bucket_usage_snapshot()
|
||||
|| snapshot.usage_snapshot_partial
|
||||
|| snapshot.usage_snapshot_converged != Some(expected_converged)
|
||||
|| snapshot.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
// Completed maintenance also covers ordinary usage. Keep its exact stored
|
||||
// proof for cache reuse, and reject mixtures of different set work proofs.
|
||||
let baseline_plan_digest = DataUsageScanPlanDigest(baseline.usage_snapshot_set_states.first()?.scan_plan_digest?);
|
||||
let baseline_plan_digest = DataUsageScanPlanDigest(snapshot.usage_snapshot_set_states.first()?.scan_plan_digest?);
|
||||
if ![
|
||||
proof.scan_plan_digest,
|
||||
scanner_bucket_work_digest(proof.scan_plan_digest, HealScanMode::Normal, true),
|
||||
@@ -195,8 +315,8 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
|
||||
for state in &baseline.usage_snapshot_set_states {
|
||||
let mut states = HashSet::with_capacity(snapshot.usage_snapshot_set_states.len());
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
|
||||
if !proof.expected_sources.contains(&source)
|
||||
|| !states.insert(source)
|
||||
@@ -213,9 +333,34 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
|
||||
(states == *proof.expected_sources).then_some(baseline_plan_digest)
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
|
||||
let authoritative = serde_json::from_slice::<DataUsageInfo>(proof.authoritative_data?).ok()?;
|
||||
if let Some(validated_digest) = complete_scanner_cache_snapshot_plan_digest(&authoritative, proof, true) {
|
||||
return Some(validated_digest);
|
||||
}
|
||||
|
||||
// A complete but superseded observation may reuse its per-set cache only
|
||||
// when it was explicitly tied to the durable authoritative baseline. It
|
||||
// remains observational: this proof grants bucket-scope reuse only and
|
||||
// never changes authoritative usage publication or dirty acknowledgement.
|
||||
let authoritative_has_identity = (crate::scanner::data_usage_info_has_persisted_baseline_identity(&authoritative)
|
||||
&& authoritative.usage_snapshot_converged != Some(false))
|
||||
|| crate::scanner::data_usage_info_is_bootstrap_pending(&authoritative);
|
||||
if !authoritative_has_identity {
|
||||
return None;
|
||||
}
|
||||
let observed = serde_json::from_slice::<DataUsageInfo>(proof.observed_candidate_data?).ok()?;
|
||||
if !observed_data_usage_is_newer(&observed, &authoritative) {
|
||||
return None;
|
||||
}
|
||||
|
||||
complete_scanner_cache_snapshot_plan_digest(&observed, proof, false)
|
||||
}
|
||||
|
||||
fn scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
dirty_buckets: HashSet<String>,
|
||||
dirty_scopes: Option<&DirtyUsageBucketScopes>,
|
||||
dirty_snapshot_complete: bool,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
@@ -237,7 +382,22 @@ fn scoped_scan_scope_from_dirty_buckets(
|
||||
return requested_scope;
|
||||
};
|
||||
|
||||
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
|
||||
let selected_bucket_prefixes = dirty_scopes
|
||||
.into_iter()
|
||||
.flat_map(|dirty_scopes| {
|
||||
selected_buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| dirty_scopes.get(bucket).map(|scope| (bucket.clone(), scope)))
|
||||
})
|
||||
.filter_map(|(bucket, scope)| match scope {
|
||||
DirtyUsageBucketScope::WholeBucket => None,
|
||||
DirtyUsageBucketScope::TopLevelEntries(entries) => {
|
||||
ScannerBucketPrefixScanScope::from_dirty_top_level_entries(entries.clone()).map(|scope| (bucket, scope))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, selected_bucket_prefixes, baseline_scan_plan_digest)
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
|
||||
@@ -286,6 +446,7 @@ pub struct ScannerBucketScanPlan {
|
||||
/// Includes mutation generations even when the set planner uses a structural digest.
|
||||
bucket_coverage_digest: DataUsageScanPlanDigest,
|
||||
requires_full_scan: bool,
|
||||
service_cohort: Option<Arc<StdMutex<ScannerServiceCohort>>>,
|
||||
// Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
|
||||
execution_digest: DataUsageScanPlanDigest,
|
||||
leader_epoch: u64,
|
||||
@@ -732,6 +893,12 @@ pub trait ScannerIOCache: Send + Sync + Debug + 'static {
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ScannerDiskScanOptions {
|
||||
pub scan_mode: HealScanMode,
|
||||
pub prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait ScannerIODisk: Send + Sync + Debug + 'static {
|
||||
async fn nsscanner_disk(
|
||||
@@ -741,7 +908,7 @@ pub trait ScannerIODisk: Send + Sync + Debug + 'static {
|
||||
set_disks: Vec<Arc<Disk>>,
|
||||
cache: DataUsageCache,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
options: ScannerDiskScanOptions,
|
||||
) -> Result<ScannerDiskScanOutcome>;
|
||||
|
||||
async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary>;
|
||||
@@ -868,6 +1035,7 @@ pub(crate) struct ScannerCycleResult {
|
||||
failed_dirty_usage: bool,
|
||||
pending_maintenance_work: bool,
|
||||
required_cycle_floor: Option<u64>,
|
||||
publication_expectation: Option<ScannerPublicationExpectation>,
|
||||
}
|
||||
|
||||
impl ScannerCycleResult {
|
||||
@@ -883,10 +1051,12 @@ impl ScannerCycleResult {
|
||||
failed_dirty_usage: false,
|
||||
pending_maintenance_work: false,
|
||||
required_cycle_floor: None,
|
||||
publication_expectation: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.publication_epoch = publication_epoch;
|
||||
self
|
||||
}
|
||||
@@ -896,6 +1066,7 @@ impl ScannerCycleResult {
|
||||
}
|
||||
|
||||
fn with_activity_digest(mut self, activity_digest: [u8; 32]) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.activity_digest = Some(activity_digest);
|
||||
self
|
||||
}
|
||||
@@ -905,6 +1076,7 @@ impl ScannerCycleResult {
|
||||
}
|
||||
|
||||
pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.observational_snapshot_published = published;
|
||||
self
|
||||
}
|
||||
@@ -914,16 +1086,19 @@ impl ScannerCycleResult {
|
||||
}
|
||||
|
||||
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.failed_dirty_usage = failed_dirty_usage;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_pending_maintenance_work(mut self, pending_maintenance_work: bool) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.pending_maintenance_work = pending_maintenance_work;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_required_cycle_floor(mut self, required_cycle_floor: Option<u64>) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.required_cycle_floor = required_cycle_floor;
|
||||
self
|
||||
}
|
||||
@@ -932,11 +1107,13 @@ impl ScannerCycleResult {
|
||||
mut self,
|
||||
acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.remote_dirty_usage_acknowledgements = acknowledgements;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_remote_publication_lease_targets(mut self, targets: Vec<(String, String, u64)>) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.remote_publication_lease_targets = targets;
|
||||
self
|
||||
}
|
||||
@@ -945,7 +1122,32 @@ impl ScannerCycleResult {
|
||||
&self.remote_publication_lease_targets
|
||||
}
|
||||
|
||||
pub(crate) fn acknowledge_durable_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
|
||||
pub(crate) fn publication_expectation(&self) -> Option<ScannerPublicationExpectation> {
|
||||
self.publication_expectation.clone()
|
||||
}
|
||||
|
||||
fn with_publication_expectation(mut self, expectation: Option<ScannerPublicationExpectation>) -> Self {
|
||||
// Seal only after all coverage and acknowledgement inputs are final.
|
||||
self.publication_expectation = expectation;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn acknowledge_durable_usage(
|
||||
self,
|
||||
proof: &crate::scanner::RootPublicationProof,
|
||||
) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
|
||||
if self.status != ScannerCycleStatus::Complete
|
||||
|| self
|
||||
.publication_expectation
|
||||
.as_ref()
|
||||
.is_none_or(|expected| proof.verified_version_for(expected).is_none())
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
self.clear_verified_usage()
|
||||
}
|
||||
|
||||
fn clear_verified_usage(self) -> Vec<crate::scanner::ScannerDirtyUsageAcknowledgement> {
|
||||
if let Some(snapshot) = self.dirty_usage_clear {
|
||||
clear_dirty_usage_buckets(&snapshot);
|
||||
}
|
||||
@@ -973,6 +1175,7 @@ impl ScannerCycleResult {
|
||||
mod cache;
|
||||
mod dirty_usage;
|
||||
mod guards;
|
||||
pub(crate) use guards::ScannerServiceCohort;
|
||||
mod io_cache;
|
||||
mod io_cycle;
|
||||
#[cfg(test)]
|
||||
@@ -984,6 +1187,7 @@ mod publish_gate_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) use cache::ScannerPublicationExpectation;
|
||||
use cache::*;
|
||||
use dirty_usage::*;
|
||||
use guards::*;
|
||||
@@ -995,8 +1199,8 @@ pub(crate) use cache::{
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -308,6 +308,86 @@ impl<'a> ValidatedScannerSnapshot<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ScannerPublicationExpectation {
|
||||
candidate: Arc<([u8; 32], DataUsageScanPlanDigest)>,
|
||||
}
|
||||
|
||||
impl ScannerPublicationExpectation {
|
||||
pub(crate) fn matches_encoded_candidate(&self, digest: &[u8; 32]) -> bool {
|
||||
&self.candidate.0 == digest
|
||||
}
|
||||
|
||||
pub(crate) fn same_candidate(&self, other: &Self) -> bool {
|
||||
Arc::ptr_eq(&self.candidate, &other.candidate) && self.candidate.1 == other.candidate.1
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ValidatedUsageCandidate {
|
||||
data: DataUsageInfo,
|
||||
#[cfg(test)]
|
||||
last_update: SystemTime,
|
||||
coverage_digest: DataUsageScanPlanDigest,
|
||||
}
|
||||
|
||||
pub(super) fn empty_namespace_usage_candidate(
|
||||
all_buckets: &[BucketInfo],
|
||||
sources: &HashSet<DataUsageCacheSource>,
|
||||
buckets_by_source: &HashMap<DataUsageCacheSource, Vec<BucketInfo>>,
|
||||
identity: ScannerSnapshotIdentity,
|
||||
) -> Option<ValidatedUsageCandidate> {
|
||||
if !all_buckets.is_empty()
|
||||
|| sources.is_empty()
|
||||
|| sources.len() != buckets_by_source.len()
|
||||
|| sources
|
||||
.iter()
|
||||
.any(|source| buckets_by_source.get(source).is_none_or(|buckets| !buckets.is_empty()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let last_update = SystemTime::now();
|
||||
Some(ValidatedUsageCandidate {
|
||||
data: DataUsageInfo {
|
||||
last_update: Some(last_update),
|
||||
scanner_cycle: Some(identity.cycle),
|
||||
scanner_epoch: Some(identity.leader_epoch),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
},
|
||||
#[cfg(test)]
|
||||
last_update,
|
||||
coverage_digest: identity.coverage_digest,
|
||||
})
|
||||
}
|
||||
|
||||
impl ValidatedUsageCandidate {
|
||||
pub(super) fn prepare(mut self, status: ScannerCycleStatus) -> (DataUsageInfo, Option<ScannerPublicationExpectation>) {
|
||||
self.data.usage_snapshot_converged = Some(status == ScannerCycleStatus::Complete);
|
||||
let expectation = if status == ScannerCycleStatus::Complete {
|
||||
struct DigestWriter(Sha256);
|
||||
impl std::io::Write for DigestWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.update(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut writer = DigestWriter(Sha256::new());
|
||||
serde_json::to_writer(&mut writer, &self.data)
|
||||
.ok()
|
||||
.map(|()| ScannerPublicationExpectation {
|
||||
candidate: Arc::new((writer.0.finalize().into(), self.coverage_digest)),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
(self.data, expectation)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn completed_data_usage_info(
|
||||
results: &[DataUsageCache],
|
||||
scope: &ScannerSnapshotScope<'_>,
|
||||
@@ -316,6 +396,18 @@ pub(super) fn completed_data_usage_info(
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
completed_usage_candidate(results, scope, tier_registry_names, bucket_plan_complete, budget_elapsed, cancelled)
|
||||
.map(|candidate| (candidate.data, candidate.last_update))
|
||||
}
|
||||
|
||||
pub(super) fn completed_usage_candidate(
|
||||
results: &[DataUsageCache],
|
||||
scope: &ScannerSnapshotScope<'_>,
|
||||
tier_registry_names: &[String],
|
||||
bucket_plan_complete: bool,
|
||||
budget_elapsed: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<ValidatedUsageCandidate> {
|
||||
if !bucket_plan_complete {
|
||||
return None;
|
||||
}
|
||||
@@ -393,7 +485,12 @@ pub(super) fn completed_data_usage_info(
|
||||
usage_snapshot_set_states,
|
||||
..Default::default()
|
||||
};
|
||||
Some((data_usage_info, merged_last_update))
|
||||
Some(ValidatedUsageCandidate {
|
||||
data: data_usage_info,
|
||||
#[cfg(test)]
|
||||
last_update: merged_last_update,
|
||||
coverage_digest: scope.identity.coverage_digest,
|
||||
})
|
||||
}
|
||||
|
||||
fn tier_accounting_proof_is_publishable(
|
||||
|
||||
@@ -16,6 +16,12 @@ use super::*;
|
||||
|
||||
pub(super) static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
// Lock order when both dirty maps are needed is `DIRTY_USAGE_BUCKETS` followed
|
||||
// by `DIRTY_USAGE_BUCKET_SCOPES`. Both are held only for synchronous map
|
||||
// updates, so no scanner task can observe a bucket generation without its
|
||||
// matching scope.
|
||||
pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock<StdMutex<DirtyUsageBucketScopes>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
@@ -33,6 +39,21 @@ pub struct ScannerDirtyUsageBucket {
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
/// A non-durable optimization hint for a dirty bucket.
|
||||
///
|
||||
/// A whole-bucket marker always wins over narrow path hints. The scanner never
|
||||
/// publishes a prefix-only result as authoritative usage; this only controls
|
||||
/// whether a complete per-bucket cache can reuse known-clean direct children.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum DirtyUsageBucketScope {
|
||||
WholeBucket,
|
||||
TopLevelEntries(HashSet<String>),
|
||||
}
|
||||
|
||||
pub(super) type DirtyUsageBucketScopes = HashMap<String, DirtyUsageBucketScope>;
|
||||
|
||||
const MAX_DIRTY_USAGE_TOP_LEVEL_ENTRIES_PER_BUCKET: usize = 128;
|
||||
|
||||
/// A point-in-time view of the local dirty bucket generations.
|
||||
///
|
||||
/// `complete == false` is an all-or-nothing overflow signal: `buckets` is
|
||||
@@ -67,6 +88,7 @@ pub fn acknowledge_scoped_dirty_usage(
|
||||
// No await or storage operation occurs while the dirty map is locked.
|
||||
let (cleared, pending) = {
|
||||
let mut dirty = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let checked = entries
|
||||
.iter()
|
||||
.map(|(guard, generation)| {
|
||||
@@ -81,6 +103,7 @@ pub fn acknowledge_scoped_dirty_usage(
|
||||
scanner_activity_epoch(),
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
|
||||
&mut dirty,
|
||||
&mut dirty_scopes,
|
||||
&checked,
|
||||
probe_only,
|
||||
)?;
|
||||
@@ -100,6 +123,7 @@ fn apply_scoped_dirty_usage_ack(
|
||||
current_instance: &str,
|
||||
current_generation: u64,
|
||||
dirty: &mut DirtyUsageBuckets,
|
||||
dirty_scopes: &mut DirtyUsageBucketScopes,
|
||||
entries: &[(&str, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
|
||||
@@ -118,6 +142,7 @@ fn apply_scoped_dirty_usage_ack(
|
||||
for (bucket, generation) in entries {
|
||||
if dirty.get(*bucket) == Some(generation) {
|
||||
dirty.remove(*bucket);
|
||||
dirty_scopes.remove(*bucket);
|
||||
cleared += 1;
|
||||
}
|
||||
}
|
||||
@@ -132,30 +157,60 @@ mod scoped_dirty_usage_tests {
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
|
||||
let mut scopes = HashMap::from([
|
||||
("hot".to_string(), DirtyUsageBucketScope::WholeBucket),
|
||||
(
|
||||
"cold".to_string(),
|
||||
DirtyUsageBucketScope::TopLevelEntries(HashSet::from(["first".to_string()])),
|
||||
),
|
||||
]);
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &mut scopes, &[("cold", 8)], true),
|
||||
Ok(0)
|
||||
);
|
||||
assert_eq!(dirty.len(), 2);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
|
||||
assert!(scopes.contains_key("cold"));
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &mut scopes, &[("cold", 8)], false),
|
||||
Ok(1)
|
||||
);
|
||||
assert_eq!(dirty.get("hot"), Some(&7));
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
assert!(!scopes.contains_key("cold"));
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &mut scopes, &[("cold", 8)], false),
|
||||
Ok(0)
|
||||
);
|
||||
dirty.insert("cold".to_string(), 9);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
scopes.insert("cold".to_string(), DirtyUsageBucketScope::WholeBucket);
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &mut scopes, &[("cold", 8)], false),
|
||||
Ok(0)
|
||||
);
|
||||
assert_eq!(dirty.get("cold"), Some(&9));
|
||||
assert!(scopes.contains_key("cold"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
|
||||
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
let mut dirty = original.clone();
|
||||
let original_scopes = HashMap::from([
|
||||
("hot".to_string(), DirtyUsageBucketScope::WholeBucket),
|
||||
("cold".to_string(), DirtyUsageBucketScope::WholeBucket),
|
||||
]);
|
||||
let mut scopes = original_scopes.clone();
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
|
||||
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &mut scopes, &[("cold", 8)], false),
|
||||
Err(ScannerDirtyUsageAckError::ProcessChanged)
|
||||
);
|
||||
assert_eq!(scopes, original_scopes);
|
||||
for generation in [0, 9, u64::MAX] {
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &mut scopes, &[("cold", 8), ("hot", generation)], false,),
|
||||
Err(ScannerDirtyUsageAckError::InvalidGeneration)
|
||||
);
|
||||
assert_eq!(dirty, original);
|
||||
assert_eq!(scopes, original_scopes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -164,6 +219,12 @@ pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn dirty_usage_bucket_scopes() -> MutexGuard<'static, DirtyUsageBucketScopes> {
|
||||
DIRTY_USAGE_BUCKET_SCOPES
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
@@ -181,8 +242,10 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_scopes.insert(bucket.to_string(), DirtyUsageBucketScope::WholeBucket);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
@@ -193,6 +256,56 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
/// Record a mutation whose affected top-level namespace entry is known.
|
||||
///
|
||||
/// Object names that cannot be represented as one safe direct child retain the
|
||||
/// conservative whole-bucket marker. This journal is intentionally process
|
||||
/// local: after restart or any unverified distributed path the scanner falls
|
||||
/// back to its ordinary bucket scan.
|
||||
pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
||||
let Some(top_level_entry) = dirty_usage_top_level_entry(object) else {
|
||||
record_dirty_usage_bucket(bucket);
|
||||
return;
|
||||
};
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
let scope = dirty_scopes
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(|| DirtyUsageBucketScope::TopLevelEntries(HashSet::new()));
|
||||
let overflowed = match scope {
|
||||
DirtyUsageBucketScope::WholeBucket => false,
|
||||
DirtyUsageBucketScope::TopLevelEntries(entries) => {
|
||||
entries.insert(top_level_entry);
|
||||
entries.len() > MAX_DIRTY_USAGE_TOP_LEVEL_ENTRIES_PER_BUCKET
|
||||
}
|
||||
};
|
||||
if overflowed {
|
||||
*scope = DirtyUsageBucketScope::WholeBucket;
|
||||
}
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
crate::prefix_usage::invalidate_prefix_usage_cache(bucket);
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
fn dirty_usage_top_level_entry(object: &str) -> Option<String> {
|
||||
let (top_level_entry, _) = object.split_once('/').unwrap_or((object, ""));
|
||||
(!top_level_entry.is_empty()
|
||||
&& top_level_entry != "."
|
||||
&& top_level_entry != ".."
|
||||
&& !object.starts_with('/')
|
||||
&& !top_level_entry.contains(['\\', '\0']))
|
||||
.then(|| top_level_entry.to_string())
|
||||
}
|
||||
|
||||
pub fn record_scanner_maintenance_change(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
@@ -262,6 +375,7 @@ pub fn acknowledge_dirty_usage_generation(
|
||||
|
||||
let (cleared_buckets, pending_buckets) = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let current_generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
if generation == 0 || generation == u64::MAX || current_generation == u64::MAX || generation > current_generation {
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
@@ -269,6 +383,7 @@ pub fn acknowledge_dirty_usage_generation(
|
||||
|
||||
let before = dirty_buckets.len();
|
||||
dirty_buckets.retain(|_, dirty_generation| *dirty_generation > generation);
|
||||
dirty_scopes.retain(|bucket, _| dirty_buckets.contains_key(bucket));
|
||||
let cleared_buckets = before.saturating_sub(dirty_buckets.len());
|
||||
if cleared_buckets > 0 {
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
@@ -291,7 +406,9 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
dirty_buckets.remove(bucket);
|
||||
dirty_scopes.remove(bucket);
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
@@ -299,8 +416,9 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
}
|
||||
|
||||
pub(super) fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_generation_cutoff: u64) -> DirtyUsageSnapshot {
|
||||
let (snapshot, generation, covers_all_pending) = {
|
||||
let (snapshot, scopes, generation, covers_all_pending) = {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
let dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let listed_buckets = dirty_buckets
|
||||
.values()
|
||||
.any(|generation| *generation > absent_generation_cutoff)
|
||||
@@ -315,13 +433,26 @@ pub(super) fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo], absent_genera
|
||||
})
|
||||
.map(|(bucket, generation)| (bucket.clone(), *generation))
|
||||
.collect::<DirtyUsageBuckets>();
|
||||
let scopes = snapshot
|
||||
.keys()
|
||||
.map(|bucket| {
|
||||
(
|
||||
bucket.clone(),
|
||||
dirty_scopes
|
||||
.get(bucket)
|
||||
.cloned()
|
||||
.unwrap_or(DirtyUsageBucketScope::WholeBucket),
|
||||
)
|
||||
})
|
||||
.collect::<DirtyUsageBucketScopes>();
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
|
||||
let covers_all_pending = generation == absent_generation_cutoff && snapshot.len() == dirty_buckets.len();
|
||||
(snapshot, generation, covers_all_pending)
|
||||
(snapshot, scopes, generation, covers_all_pending)
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
|
||||
DirtyUsageSnapshot {
|
||||
buckets: Arc::new(snapshot),
|
||||
scopes: Arc::new(scopes),
|
||||
generation,
|
||||
covers_all_pending,
|
||||
}
|
||||
@@ -338,10 +469,12 @@ pub(crate) async fn dirty_usage_bucket_notified() {
|
||||
pub(super) fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
|
||||
let (cleared_buckets, pending_buckets) = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
let mut cleared_buckets = 0usize;
|
||||
for (bucket, generation) in snapshot {
|
||||
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
|
||||
dirty_buckets.remove(bucket);
|
||||
dirty_scopes.remove(bucket);
|
||||
cleared_buckets += 1;
|
||||
}
|
||||
}
|
||||
@@ -443,9 +576,15 @@ pub(super) fn dirty_usage_bucket_count() -> usize {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
dirty_usage_bucket_scopes().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn dirty_usage_buckets_for_tests() -> DirtyUsageBuckets {
|
||||
dirty_usage_buckets().clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn dirty_usage_bucket_scopes_for_tests() -> DirtyUsageBucketScopes {
|
||||
dirty_usage_bucket_scopes().clone()
|
||||
}
|
||||
|
||||
@@ -14,6 +14,312 @@
|
||||
/// scan concurrency accounting: gauge recorders, RAII guards, and worker limits.
|
||||
use super::*;
|
||||
|
||||
const SCANNER_SERVICE_COHORT_MAX_MEMBERS: usize = 4096;
|
||||
const SCANNER_SERVICE_COHORT_MAX_NAME_BYTES: usize = 128 * 1024;
|
||||
|
||||
static SERVICE_COHORT_METRICS_OWNER: StdMutex<std::sync::Weak<()>> = StdMutex::new(std::sync::Weak::new());
|
||||
|
||||
struct ScannerCohortMetricsOwner(Arc<()>);
|
||||
|
||||
impl Default for ScannerCohortMetricsOwner {
|
||||
fn default() -> Self {
|
||||
let owner = Arc::new(());
|
||||
let mut current = SERVICE_COHORT_METRICS_OWNER
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
*current = Arc::downgrade(&owner);
|
||||
write_service_cohort_metrics(0, 0.0, false);
|
||||
Self(owner)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerCohortMetricsOwner {
|
||||
fn drop(&mut self) {
|
||||
let mut current = SERVICE_COHORT_METRICS_OWNER
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if current.ptr_eq(&Arc::downgrade(&self.0)) {
|
||||
*current = std::sync::Weak::new();
|
||||
write_service_cohort_metrics(0, 0.0, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_service_cohort_metrics(waiting: usize, oldest: f64, overflowed: bool) {
|
||||
metrics::gauge!("rustfs_scanner_service_cohort_waiting").set(waiting as f64);
|
||||
metrics::gauge!("rustfs_scanner_service_cohort_oldest_wait_seconds").set(oldest);
|
||||
metrics::gauge!("rustfs_scanner_service_cohort_capacity_fallback").set(if overflowed { 1.0 } else { 0.0 });
|
||||
}
|
||||
|
||||
struct ScannerCohortWait {
|
||||
order: u64,
|
||||
queued_at: Instant,
|
||||
admitted: bool,
|
||||
present: bool,
|
||||
}
|
||||
|
||||
/// Leader-local admission order, never evidence of completed scan coverage.
|
||||
/// Retains at most 4096 members and 128 KiB of name payload, including the
|
||||
/// cursor shared with its last member. Candidate selection borrows at most
|
||||
/// 4096 inventory entries; the existing full inventory is not bounded here.
|
||||
pub(crate) struct ScannerServiceCohort {
|
||||
members: HashMap<DataUsageCacheSource, HashMap<Arc<str>, ScannerCohortWait>>,
|
||||
cursor: Option<(DataUsageCacheSource, Arc<str>)>,
|
||||
next_order: u64,
|
||||
max_members: usize,
|
||||
max_name_bytes: usize,
|
||||
overflowed: bool,
|
||||
metrics_owner: ScannerCohortMetricsOwner,
|
||||
waiting: usize,
|
||||
oldest_wait_at_refresh: f64,
|
||||
#[cfg(test)]
|
||||
metric_members_examined: usize,
|
||||
}
|
||||
|
||||
impl Default for ScannerServiceCohort {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
members: HashMap::new(),
|
||||
cursor: None,
|
||||
next_order: 0,
|
||||
max_members: SCANNER_SERVICE_COHORT_MAX_MEMBERS,
|
||||
max_name_bytes: SCANNER_SERVICE_COHORT_MAX_NAME_BYTES,
|
||||
overflowed: false,
|
||||
metrics_owner: ScannerCohortMetricsOwner::default(),
|
||||
waiting: 0,
|
||||
oldest_wait_at_refresh: 0.0,
|
||||
#[cfg(test)]
|
||||
metric_members_examined: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ScannerServiceCohort {
|
||||
pub(crate) fn refresh(&mut self, inventory: &HashMap<DataUsageCacheSource, Vec<BucketInfo>>) {
|
||||
let count = inventory
|
||||
.values()
|
||||
.fold(0usize, |count, buckets| count.saturating_add(buckets.len()));
|
||||
let name_bytes = inventory
|
||||
.values()
|
||||
.flatten()
|
||||
.fold(0usize, |bytes, bucket| bytes.saturating_add(bucket.name.len()));
|
||||
self.overflowed = count > self.max_members || name_bytes > self.max_name_bytes;
|
||||
for wait in self.members.values_mut().flat_map(HashMap::values_mut) {
|
||||
wait.present = false;
|
||||
}
|
||||
for (source, buckets) in inventory {
|
||||
for bucket in buckets {
|
||||
if let Some(wait) = self
|
||||
.members
|
||||
.get_mut(source)
|
||||
.and_then(|members| members.get_mut(bucket.name.as_str()))
|
||||
{
|
||||
wait.present = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.members.retain(|_, buckets| {
|
||||
buckets.retain(|_, wait| wait.present);
|
||||
!buckets.is_empty()
|
||||
});
|
||||
if self.members.values().flat_map(HashMap::values).all(|wait| wait.admitted) {
|
||||
self.members.clear();
|
||||
self.next_order = 0;
|
||||
}
|
||||
if count == 0 {
|
||||
self.cursor = None;
|
||||
}
|
||||
let mut member_count = self.members.values().map(HashMap::len).sum::<usize>();
|
||||
let mut retained_bytes = self
|
||||
.members
|
||||
.values()
|
||||
.flat_map(HashMap::keys)
|
||||
.map(|name| name.len())
|
||||
.sum::<usize>();
|
||||
let mut incoming = self.admission_candidates(inventory, true);
|
||||
if incoming.is_empty() {
|
||||
incoming = self.admission_candidates(inventory, false);
|
||||
}
|
||||
for (pool, set, bucket) in incoming {
|
||||
if member_count >= self.max_members {
|
||||
break;
|
||||
}
|
||||
if retained_bytes.saturating_add(bucket.len()) > self.max_name_bytes {
|
||||
continue;
|
||||
}
|
||||
let Some(next_order) = self.next_order.checked_add(1) else {
|
||||
self.overflowed = true;
|
||||
break;
|
||||
};
|
||||
let source = DataUsageCacheSource::new(pool, set);
|
||||
let members = self.members.entry(source).or_default();
|
||||
if members.contains_key(bucket) {
|
||||
continue;
|
||||
}
|
||||
let name: Arc<str> = bucket.into();
|
||||
retained_bytes += name.len();
|
||||
member_count += 1;
|
||||
members.insert(
|
||||
name.clone(),
|
||||
ScannerCohortWait {
|
||||
order: self.next_order,
|
||||
queued_at: Instant::now(),
|
||||
admitted: false,
|
||||
present: true,
|
||||
},
|
||||
);
|
||||
self.cursor = Some((source, name));
|
||||
self.next_order = next_order;
|
||||
}
|
||||
self.refresh_metrics();
|
||||
}
|
||||
|
||||
fn admission_candidates<'a>(
|
||||
&self,
|
||||
inventory: &'a HashMap<DataUsageCacheSource, Vec<BucketInfo>>,
|
||||
after_cursor: bool,
|
||||
) -> Vec<(usize, usize, &'a str)> {
|
||||
let mut candidates = std::collections::BinaryHeap::new();
|
||||
for (source, buckets) in inventory {
|
||||
for bucket in buckets {
|
||||
let key = (source.pool_index, source.set_index, bucket.name.as_str());
|
||||
let after = self
|
||||
.cursor
|
||||
.as_ref()
|
||||
.is_none_or(|(source, name)| key > (source.pool_index, source.set_index, name.as_ref()));
|
||||
if after != after_cursor
|
||||
|| bucket.name.len() > self.max_name_bytes
|
||||
|| self
|
||||
.members
|
||||
.get(source)
|
||||
.is_some_and(|members| members.contains_key(bucket.name.as_str()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if candidates.len() < self.max_members {
|
||||
candidates.push(key);
|
||||
} else if candidates.peek().is_some_and(|last| key < *last) {
|
||||
candidates.pop();
|
||||
candidates.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
candidates.into_sorted_vec()
|
||||
}
|
||||
|
||||
pub(crate) fn order_set_indices(&self, sets: &[Arc<SetDisks>]) -> Vec<usize> {
|
||||
let ranks = self
|
||||
.members
|
||||
.iter()
|
||||
.map(|(source, buckets)| {
|
||||
(
|
||||
*source,
|
||||
buckets
|
||||
.values()
|
||||
.filter(|wait| !wait.admitted)
|
||||
.map(|wait| wait.order)
|
||||
.min()
|
||||
.unwrap_or(u64::MAX),
|
||||
)
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut indices = (0..sets.len()).collect::<Vec<_>>();
|
||||
indices.sort_by_key(|index| {
|
||||
(
|
||||
ranks
|
||||
.get(&DataUsageCacheSource::new(sets[*index].pool_index, sets[*index].set_index))
|
||||
.copied()
|
||||
.unwrap_or(u64::MAX),
|
||||
*index,
|
||||
)
|
||||
});
|
||||
indices
|
||||
}
|
||||
|
||||
pub(crate) fn order_buckets(&self, source: DataUsageCacheSource, buckets: &mut [BucketInfo]) {
|
||||
let rank = |bucket: &str| {
|
||||
self.members
|
||||
.get(&source)
|
||||
.and_then(|members| members.get(bucket))
|
||||
.filter(|wait| !wait.admitted)
|
||||
.map_or(u64::MAX, |wait| wait.order)
|
||||
};
|
||||
// Stable sorting preserves the existing dispatch order in the tail.
|
||||
buckets.sort_by_key(|bucket| rank(&bucket.name));
|
||||
}
|
||||
|
||||
pub(crate) fn record_admitted(&mut self, source: DataUsageCacheSource, bucket: &str) {
|
||||
let Some(wait) = self.members.get_mut(&source).and_then(|members| members.get_mut(bucket)) else {
|
||||
return;
|
||||
};
|
||||
if wait.admitted {
|
||||
return;
|
||||
}
|
||||
wait.admitted = true;
|
||||
self.waiting -= 1;
|
||||
if self.waiting == 0 {
|
||||
self.oldest_wait_at_refresh = 0.0;
|
||||
}
|
||||
self.record_metrics();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn admitted_members(&self) -> Vec<(DataUsageCacheSource, String)> {
|
||||
self.members
|
||||
.iter()
|
||||
.flat_map(|(source, buckets)| {
|
||||
buckets
|
||||
.iter()
|
||||
.filter(|(_, wait)| wait.admitted)
|
||||
.map(|(bucket, _)| (*source, bucket.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn refresh_metrics(&mut self) {
|
||||
// Oldest age is an inventory-refresh snapshot, not a per-admission
|
||||
// scan of the cohort. Clear it immediately when no waiters remain.
|
||||
let (mut waiting, mut oldest) = (0usize, 0.0f64);
|
||||
for wait in self.members.values().flat_map(HashMap::values) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.metric_members_examined += 1;
|
||||
}
|
||||
if !wait.admitted {
|
||||
waiting += 1;
|
||||
oldest = oldest.max(wait.queued_at.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
self.waiting = waiting;
|
||||
self.oldest_wait_at_refresh = oldest;
|
||||
self.record_metrics();
|
||||
}
|
||||
|
||||
fn record_metrics(&self) {
|
||||
// Serialize owner replacement, publication and retirement. A retired
|
||||
// scanner must neither publish nor clear a replacement's gauges.
|
||||
let current = SERVICE_COHORT_METRICS_OWNER
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if current.ptr_eq(&Arc::downgrade(&self.metrics_owner.0)) {
|
||||
write_service_cohort_metrics(self.waiting, self.oldest_wait_at_refresh, self.overflowed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_bucket_scan_permit(
|
||||
semaphore: &Arc<Semaphore>,
|
||||
ctx: &CancellationToken,
|
||||
complete: &CancellationToken,
|
||||
) -> Option<tokio::sync::OwnedSemaphorePermit> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = complete.cancelled() => None,
|
||||
_ = ctx.cancelled() => None,
|
||||
permit = semaphore.clone().acquire_owned() => permit.ok(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn bucket_usage_scan_order(
|
||||
buckets: &[BucketInfo],
|
||||
old_cache: &DataUsageCache,
|
||||
@@ -288,6 +594,340 @@ mod tests {
|
||||
use rustfs_scanner_metrics::metrics::{ScannerWorkSource, global_metrics};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordedGauge(AtomicU64);
|
||||
|
||||
impl metrics::GaugeFn for RecordedGauge {
|
||||
fn increment(&self, value: f64) {
|
||||
self.set(f64::from_bits(self.0.load(Ordering::Relaxed)) + value);
|
||||
}
|
||||
fn decrement(&self, value: f64) {
|
||||
self.increment(-value);
|
||||
}
|
||||
fn set(&self, value: f64) {
|
||||
self.0.store(value.to_bits(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CohortGaugeRecorder(StdMutex<HashMap<String, Arc<RecordedGauge>>>);
|
||||
|
||||
impl metrics::Recorder for CohortGaugeRecorder {
|
||||
fn describe_counter(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_gauge(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_histogram(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn register_counter(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter {
|
||||
metrics::Counter::noop()
|
||||
}
|
||||
fn register_histogram(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Histogram {
|
||||
metrics::Histogram::noop()
|
||||
}
|
||||
fn register_gauge(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge {
|
||||
metrics::Gauge::from_arc(
|
||||
self.0
|
||||
.lock()
|
||||
.expect("gauge recorder")
|
||||
.entry(key.name().to_string())
|
||||
.or_default()
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl CohortGaugeRecorder {
|
||||
fn value(&self, name: &str) -> f64 {
|
||||
f64::from_bits(self.0.lock().expect("gauge recorder")[name].0.load(Ordering::Relaxed))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_metrics_retire_only_the_current_owner() {
|
||||
let recorder = CohortGaugeRecorder::default();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let mut old = ScannerServiceCohort::default();
|
||||
old.refresh(&cohort_inventory(&["old"]));
|
||||
let mut current = ScannerServiceCohort {
|
||||
max_members: 1,
|
||||
..Default::default()
|
||||
};
|
||||
current.refresh(&cohort_inventory(&["a", "b"]));
|
||||
for wait in current.members.values_mut().flat_map(HashMap::values_mut) {
|
||||
wait.queued_at = Instant::now() - Duration::from_secs(60);
|
||||
}
|
||||
current.refresh_metrics();
|
||||
old.refresh(&cohort_inventory(&["old", "more"]));
|
||||
drop(old);
|
||||
assert_eq!(recorder.value("rustfs_scanner_service_cohort_waiting"), 1.0);
|
||||
assert_eq!(recorder.value("rustfs_scanner_service_cohort_capacity_fallback"), 1.0);
|
||||
assert!(recorder.value("rustfs_scanner_service_cohort_oldest_wait_seconds") >= 60.0);
|
||||
drop(current);
|
||||
for metric in [
|
||||
"rustfs_scanner_service_cohort_waiting",
|
||||
"rustfs_scanner_service_cohort_oldest_wait_seconds",
|
||||
"rustfs_scanner_service_cohort_capacity_fallback",
|
||||
] {
|
||||
assert_eq!(recorder.value(metric), 0.0, "owner retirement must clear {metric}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_admission_metrics_do_not_rescan_a_full_window() {
|
||||
let recorder = CohortGaugeRecorder::default();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let names = (0..SCANNER_SERVICE_COHORT_MAX_MEMBERS)
|
||||
.map(|index| format!("bucket-{index:04}"))
|
||||
.collect::<Vec<_>>();
|
||||
let inventory = cohort_inventory(&names.iter().map(String::as_str).collect::<Vec<_>>());
|
||||
let mut cohort = ScannerServiceCohort::default();
|
||||
cohort.refresh(&inventory);
|
||||
assert_eq!(cohort.metric_members_examined, SCANNER_SERVICE_COHORT_MAX_MEMBERS);
|
||||
assert_eq!(cohort.waiting, SCANNER_SERVICE_COHORT_MAX_MEMBERS);
|
||||
for (index, name) in names.iter().enumerate() {
|
||||
cohort.record_admitted(source, name);
|
||||
for _ in 0..10 {
|
||||
cohort.record_admitted(source, name);
|
||||
cohort.record_admitted(source, "untracked-overflow-name");
|
||||
cohort.record_admitted(DataUsageCacheSource::new(99, 0), name);
|
||||
}
|
||||
assert_eq!(cohort.waiting, SCANNER_SERVICE_COHORT_MAX_MEMBERS - index - 1);
|
||||
assert_eq!(
|
||||
cohort.metric_members_examined, SCANNER_SERVICE_COHORT_MAX_MEMBERS,
|
||||
"tracked, repeated and overflow admissions must not scan cohort members"
|
||||
);
|
||||
}
|
||||
assert_eq!(recorder.value("rustfs_scanner_service_cohort_waiting"), 0.0);
|
||||
assert_eq!(recorder.value("rustfs_scanner_service_cohort_oldest_wait_seconds"), 0.0);
|
||||
cohort.refresh(&inventory);
|
||||
assert_eq!(
|
||||
cohort.metric_members_examined,
|
||||
2 * SCANNER_SERVICE_COHORT_MAX_MEMBERS,
|
||||
"one inventory refresh performs one metrics traversal"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn service_cohort_queued_permit_cancel_and_drop_return_the_same_capacity() {
|
||||
let semaphore = Arc::new(Semaphore::new(1));
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let mut cohort = ScannerServiceCohort::default();
|
||||
cohort.refresh(&cohort_inventory(&["waiting"]));
|
||||
let gauge_reset = DiskBucketScanGaugeReset::new("cohort-wait".to_string(), "0".to_string());
|
||||
record_disk_bucket_scans_queued(1, "cohort-wait", "0");
|
||||
record_disk_bucket_scans_active(0, "cohort-wait", "0");
|
||||
for cancel in [true, false] {
|
||||
let held = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("hold the sole permit as a barrier");
|
||||
let ctx = CancellationToken::new();
|
||||
let complete = CancellationToken::new();
|
||||
let mut waiter = Box::pin(wait_for_bucket_scan_permit(&semaphore, &ctx, &complete));
|
||||
assert!(
|
||||
futures::poll!(&mut waiter).is_pending(),
|
||||
"the production wait must actually enqueue behind the barrier"
|
||||
);
|
||||
assert_eq!(semaphore.available_permits(), 0);
|
||||
if cancel {
|
||||
ctx.cancel();
|
||||
assert!(waiter.as_mut().await.is_none());
|
||||
}
|
||||
drop(waiter);
|
||||
assert_eq!(semaphore.available_permits(), 0, "cancelling a waiter must not release the held permit");
|
||||
assert!(cohort.admitted_members().is_empty());
|
||||
drop(held);
|
||||
assert_eq!(semaphore.available_permits(), 1, "no queued waiter may leak or steal released capacity");
|
||||
}
|
||||
let ctx = CancellationToken::new();
|
||||
let complete = CancellationToken::new();
|
||||
let permit = wait_for_bucket_scan_permit(&semaphore, &ctx, &complete)
|
||||
.await
|
||||
.expect("same semaphore remains usable");
|
||||
let active_guard = DiskBucketScanActiveGuard::new(active.clone(), "cohort-wait".to_string(), "0".to_string());
|
||||
assert_eq!(active.load(Ordering::Relaxed), 1);
|
||||
drop(active_guard);
|
||||
drop(permit);
|
||||
drop(gauge_reset);
|
||||
assert_eq!(active.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
let state = global_metrics()
|
||||
.scanner_runtime_details_report()
|
||||
.disk_bucket_scan_states
|
||||
.into_iter()
|
||||
.find(|state| state.pool == "cohort-wait" && state.set == "0")
|
||||
.expect("fixture gauges");
|
||||
assert_eq!((state.queued, state.active), (0, 0));
|
||||
assert!(cohort.admitted_members().is_empty(), "permit ownership alone does not admit a bucket");
|
||||
}
|
||||
|
||||
fn cohort_inventory(names: &[&str]) -> HashMap<DataUsageCacheSource, Vec<BucketInfo>> {
|
||||
HashMap::from([(
|
||||
DataUsageCacheSource::new(0, 0),
|
||||
names
|
||||
.iter()
|
||||
.map(|name| BucketInfo {
|
||||
name: (*name).to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
)])
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_visits_fixed_members_within_service_round_bound() {
|
||||
let inventory = cohort_inventory(&["a", "b", "c", "d", "e"]);
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut cohort = ScannerServiceCohort::default();
|
||||
let mut admitted = HashSet::new();
|
||||
for _ in 0..3 {
|
||||
cohort.refresh(&inventory);
|
||||
let mut buckets = inventory[&source].clone();
|
||||
cohort.order_buckets(source, &mut buckets);
|
||||
for bucket in buckets.iter().take(2) {
|
||||
admitted.insert(bucket.name.clone());
|
||||
cohort.record_admitted(source, &bucket.name);
|
||||
}
|
||||
}
|
||||
assert_eq!(admitted.len(), 5, "ceil(5/2) service rounds must include every member");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_keeps_waiting_bootstrap_ahead_of_new_work() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut cohort = ScannerServiceCohort::default();
|
||||
cohort.refresh(&cohort_inventory(&["a-hot", "z-bootstrap"]));
|
||||
cohort.record_admitted(source, "a-hot");
|
||||
let queued_at = cohort.members[&source]["z-bootstrap"].queued_at;
|
||||
let inventory = cohort_inventory(&["a-hot", "aaa-new-bootstrap", "z-bootstrap"]);
|
||||
for _ in 0..10 {
|
||||
cohort.refresh(&inventory);
|
||||
let mut buckets = inventory[&source].clone();
|
||||
cohort.order_buckets(source, &mut buckets);
|
||||
assert_eq!(buckets[0].name, "z-bootstrap");
|
||||
assert_eq!(buckets[1].name, "aaa-new-bootstrap");
|
||||
assert_eq!(cohort.members[&source]["z-bootstrap"].queued_at, queued_at);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_overflow_preserves_waiters_and_rotates_finished_windows() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut cohort = ScannerServiceCohort {
|
||||
max_members: 2,
|
||||
max_name_bytes: 4,
|
||||
..Default::default()
|
||||
};
|
||||
cohort.refresh(&cohort_inventory(&["aa", "bb"]));
|
||||
cohort.record_admitted(source, "aa");
|
||||
for names in [["aa", "bb", "c"], ["aa", "bb", "d"]] {
|
||||
let inventory = cohort_inventory(&names);
|
||||
cohort.refresh(&inventory);
|
||||
assert!(cohort.overflowed);
|
||||
assert_eq!(cohort.members[&source].len(), 2);
|
||||
assert!(cohort.members[&source].contains_key("aa"));
|
||||
let mut fallback = inventory[&source].clone();
|
||||
fallback.reverse();
|
||||
cohort.order_buckets(source, &mut fallback);
|
||||
assert_eq!(fallback[0].name, "bb", "overflow must not discard a waiting member's priority");
|
||||
assert_eq!(fallback.len(), 3, "unknown tail must remain dispatchable");
|
||||
}
|
||||
cohort.record_admitted(source, "bb");
|
||||
let inventory = cohort_inventory(&["aa", "bb", "c", "d"]);
|
||||
cohort.refresh(&inventory);
|
||||
assert_eq!(
|
||||
cohort.members[&source].keys().map(AsRef::as_ref).collect::<HashSet<&str>>(),
|
||||
HashSet::from(["c", "d"])
|
||||
);
|
||||
cohort.record_admitted(source, "c");
|
||||
cohort.record_admitted(source, "d");
|
||||
cohort.refresh(&inventory);
|
||||
assert!(
|
||||
cohort.members[&source].contains_key("aa"),
|
||||
"finite inventory must wrap after the last window"
|
||||
);
|
||||
cohort.refresh(&cohort_inventory(&["bb"]));
|
||||
assert!(!cohort.overflowed);
|
||||
assert_eq!(cohort.members[&source].len(), 1);
|
||||
cohort.next_order = u64::MAX;
|
||||
cohort.refresh(&cohort_inventory(&["bb", "c"]));
|
||||
assert!(cohort.overflowed);
|
||||
assert_eq!(cohort.members[&source].len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_overflow_eventually_services_every_stable_member() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let inventory = cohort_inventory(&["aa", "bb", "cc", "dd", "ee", "ff"]);
|
||||
let mut cohort = ScannerServiceCohort {
|
||||
max_members: 2,
|
||||
max_name_bytes: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let mut admitted = HashSet::new();
|
||||
for _ in 0..3 {
|
||||
cohort.refresh(&inventory);
|
||||
let mut buckets = inventory[&source].clone();
|
||||
cohort.order_buckets(source, &mut buckets);
|
||||
for bucket in buckets.iter().take(2) {
|
||||
admitted.insert(bucket.name.clone());
|
||||
cohort.record_admitted(source, &bucket.name);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
admitted,
|
||||
HashSet::from([
|
||||
"aa".to_string(),
|
||||
"bb".to_string(),
|
||||
"cc".to_string(),
|
||||
"dd".to_string(),
|
||||
"ee".to_string(),
|
||||
"ff".to_string(),
|
||||
]),
|
||||
"stable overflow inventory must rotate every member through the tracked window"
|
||||
);
|
||||
assert!(cohort.overflowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn service_cohort_bounds_names_and_does_not_reset_duplicate_dirty_age() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let mut cohort = ScannerServiceCohort {
|
||||
max_members: 2,
|
||||
max_name_bytes: 4,
|
||||
..Default::default()
|
||||
};
|
||||
cohort.refresh(&cohort_inventory(&["aa", "bb", "long-name"]));
|
||||
let queued_at = cohort.members[&source]["bb"].queued_at;
|
||||
for _ in 0..10 {
|
||||
cohort.refresh(&cohort_inventory(&["aa", "aa", "bb", "long-name"]));
|
||||
assert_eq!(cohort.members.values().map(HashMap::len).sum::<usize>(), 2);
|
||||
assert_eq!(
|
||||
cohort
|
||||
.members
|
||||
.values()
|
||||
.flat_map(HashMap::keys)
|
||||
.map(|name| name.len())
|
||||
.sum::<usize>(),
|
||||
4
|
||||
);
|
||||
assert_eq!(cohort.members[&source]["bb"].queued_at, queued_at);
|
||||
}
|
||||
cohort.refresh(&cohort_inventory(&[]));
|
||||
assert!(cohort.members.is_empty());
|
||||
assert!(cohort.cursor.is_none());
|
||||
}
|
||||
|
||||
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
|
||||
global_metrics()
|
||||
.scanner_runtime_details_report()
|
||||
|
||||
@@ -117,6 +117,7 @@ impl ScannerIOCache for SetDisks {
|
||||
digest: scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
requires_full_scan,
|
||||
service_cohort,
|
||||
execution_digest,
|
||||
leader_epoch,
|
||||
tier_registry_generation,
|
||||
@@ -500,7 +501,13 @@ impl ScannerIOCache for SetDisks {
|
||||
|
||||
let mut permutes = buckets.clone();
|
||||
permutes.shuffle(&mut rand::rng());
|
||||
let scan_order = bucket_usage_scan_order(&permutes, &old_cache, &dirty_usage_buckets);
|
||||
let mut scan_order = bucket_usage_scan_order(&permutes, &old_cache, &dirty_usage_buckets);
|
||||
if let Some(cohort) = &service_cohort {
|
||||
cohort
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.order_buckets(source, &mut scan_order);
|
||||
}
|
||||
|
||||
for bucket in scan_order.iter() {
|
||||
if let Some(c) = old_cache.find(&bucket.name) {
|
||||
@@ -558,6 +565,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let remaining_bucket_work = Arc::new(AtomicUsize::new(buckets.len()));
|
||||
let bucket_work_complete = CancellationToken::new();
|
||||
for (disk, worker_mode) in workers {
|
||||
let service_cohort_clone = service_cohort.clone();
|
||||
let bucket_rx_mutex_clone = bucket_rx_mutex.clone();
|
||||
let bucket_tx_clone = bucket_tx.clone();
|
||||
let remaining_bucket_work_clone = remaining_bucket_work.clone();
|
||||
@@ -577,6 +585,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let partial_dirty_buckets_clone = bucket_failures.partial.clone();
|
||||
let pending_maintenance_work_clone = pending_maintenance_work.clone();
|
||||
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
|
||||
let scope_clone = scope.clone();
|
||||
let cache_cycle_floor_clone = cache_cycle_floor.clone();
|
||||
let expected_publication_epoch_clone = expected_publication_epoch;
|
||||
let remote_server_epoch = match worker_mode {
|
||||
@@ -587,6 +596,18 @@ impl ScannerIOCache for SetDisks {
|
||||
let remote_session_id = uuid::Uuid::new_v4();
|
||||
let mut remote_session_sequence = 0_u64;
|
||||
loop {
|
||||
// Do not prefetch a FIFO member into an independently
|
||||
// scheduled permit waiter: that can reorder admissions.
|
||||
let permit_wait_start = Instant::now();
|
||||
let Some(_permit) =
|
||||
wait_for_bucket_scan_permit(&disk_scan_semaphore_clone, &ctx_clone, &bucket_work_complete_clone).await
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if ctx_clone.is_cancelled() || budget_clone.budget_elapsed() {
|
||||
break;
|
||||
}
|
||||
let permit_wait_elapsed = permit_wait_start.elapsed();
|
||||
let bucket = tokio::select! {
|
||||
_ = bucket_work_complete_clone.cancelled() => break,
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
@@ -599,42 +620,34 @@ impl ScannerIOCache for SetDisks {
|
||||
};
|
||||
let mut work_guard =
|
||||
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
|
||||
// Prefix hints are process-local. Never hand one to a
|
||||
// remote or legacy-coordinator disk path.
|
||||
let prefix_scan_scope = disk_clone
|
||||
.is_local()
|
||||
.then(|| scope_clone.prefix_scope_for(&bucket.name))
|
||||
.flatten();
|
||||
|
||||
let permit_wait = ctx_clone.clone();
|
||||
let permit_wait_start = Instant::now();
|
||||
let _permit = tokio::select! {
|
||||
permit = disk_scan_semaphore_clone.clone().acquire_owned() => match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
decrement_disk_bucket_scans_queued(
|
||||
&queued_disk_bucket_scans_clone,
|
||||
&pool_label_clone,
|
||||
&set_label_clone,
|
||||
);
|
||||
break;
|
||||
},
|
||||
},
|
||||
_ = permit_wait.cancelled() => {
|
||||
decrement_disk_bucket_scans_queued(
|
||||
&queued_disk_bucket_scans_clone,
|
||||
&pool_label_clone,
|
||||
&set_label_clone,
|
||||
);
|
||||
break;
|
||||
},
|
||||
};
|
||||
metrics::histogram!(
|
||||
METRIC_SCANNER_DISK_SCAN_WAIT_SECONDS,
|
||||
"pool" => pool_label_clone.clone(),
|
||||
"set" => set_label_clone.clone()
|
||||
)
|
||||
.record(permit_wait_start.elapsed().as_secs_f64());
|
||||
.record(permit_wait_elapsed.as_secs_f64());
|
||||
decrement_disk_bucket_scans_queued(&queued_disk_bucket_scans_clone, &pool_label_clone, &set_label_clone);
|
||||
let _active_guard = DiskBucketScanActiveGuard::new(
|
||||
active_disk_bucket_scans_clone.clone(),
|
||||
pool_label_clone.clone(),
|
||||
set_label_clone.clone(),
|
||||
);
|
||||
if ctx_clone.is_cancelled() || budget_clone.budget_elapsed() {
|
||||
break;
|
||||
}
|
||||
if let Some(cohort) = &service_cohort_clone {
|
||||
cohort
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.record_admitted(source, &bucket.name);
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1048,7 +1061,10 @@ impl ScannerIOCache for SetDisks {
|
||||
set_disk_inventory_clone.as_ref().clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
scan_mode,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
},
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
|
||||
|
||||
@@ -72,7 +72,9 @@ where
|
||||
scan_mode,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: None,
|
||||
observed_usage_candidate: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
#[cfg(test)]
|
||||
resolved_scope_observer: None,
|
||||
};
|
||||
@@ -88,52 +90,79 @@ pub(crate) struct ScannerCycleRequest {
|
||||
pub(crate) scan_mode: HealScanMode,
|
||||
pub(crate) scan_scope: ScannerBucketScanScope,
|
||||
pub(crate) persisted_usage_baseline: Option<Bytes>,
|
||||
pub(crate) observed_usage_candidate: Option<Bytes>,
|
||||
/// Scheduled maintenance must visit clean buckets even with a valid dirty scope.
|
||||
pub(crate) requires_full_scan: bool,
|
||||
pub(crate) service_cohort: Option<Arc<StdMutex<ScannerServiceCohort>>>,
|
||||
#[cfg(test)]
|
||||
pub(crate) resolved_scope_observer: Option<tokio::sync::oneshot::Sender<ScannerBucketScanScope>>,
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolution<'a> {
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
baseline_proof: ScannerCacheBaselineProof<'a>,
|
||||
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
|
||||
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
|
||||
all_buckets: &'a [BucketInfo],
|
||||
requires_full_scan: bool,
|
||||
pub(super) struct ScannerBucketScopeResolution<'a> {
|
||||
pub(super) requested_scope: ScannerBucketScanScope,
|
||||
pub(super) baseline_proof: ScannerCacheBaselineProof<'a>,
|
||||
pub(super) activity_before: &'a crate::scanner::ScannerActivitySnapshot,
|
||||
pub(super) dirty_usage_snapshot: &'a DirtyUsageSnapshot,
|
||||
pub(super) all_buckets: &'a [BucketInfo],
|
||||
pub(super) requires_full_scan: bool,
|
||||
#[cfg(test)]
|
||||
pub(super) test_peer_snapshots: Option<Vec<(String, crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot)>>,
|
||||
#[cfg(test)]
|
||||
pub(super) test_scoped_dirty_usage_capability: Option<bool>,
|
||||
}
|
||||
|
||||
async fn resolve_scanner_bucket_scan_scope<S>(
|
||||
store: &S,
|
||||
distributed: bool,
|
||||
resolution: ScannerBucketScopeResolution<'_>,
|
||||
) -> ScannerBucketScanScope
|
||||
) -> ScannerBucketScopeResolutionResult
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
};
|
||||
if resolution.requires_full_scan {
|
||||
return ScannerBucketScanScope::default();
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
if !resolution.requested_scope.is_default()
|
||||
|| !resolution.dirty_usage_snapshot.covers_all_pending
|
||||
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|
||||
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
|
||||
{
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
}
|
||||
|
||||
let mut dirty_buckets = resolution
|
||||
let dirty_buckets = resolution
|
||||
.dirty_usage_snapshot
|
||||
.buckets
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
if distributed {
|
||||
let Some(notification_system) = store.scanner_notification_system() else {
|
||||
return resolution.requested_scope;
|
||||
let notification_system = store.scanner_notification_system();
|
||||
#[cfg(test)]
|
||||
let peer_snapshots = if let Some(peer_snapshots) = resolution.test_peer_snapshots.clone() {
|
||||
peer_snapshots
|
||||
} else {
|
||||
let Some(notification_system) = notification_system.as_ref() else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
peer_snapshots
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return resolution.requested_scope;
|
||||
#[cfg(not(test))]
|
||||
let peer_snapshots = {
|
||||
let Some(notification_system) = notification_system.as_ref() else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
peer_snapshots
|
||||
};
|
||||
let mut expected_peers = HashMap::new();
|
||||
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
|
||||
@@ -141,10 +170,10 @@ where
|
||||
let Some((activity_instance_id, generation, pending)) =
|
||||
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
|
||||
else {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
|
||||
return resolution.requested_scope;
|
||||
return default_result(resolution.requested_scope);
|
||||
}
|
||||
expected_peers.insert(
|
||||
host,
|
||||
@@ -155,19 +184,82 @@ where
|
||||
},
|
||||
);
|
||||
}
|
||||
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
|
||||
return resolution.requested_scope;
|
||||
let Some(remote_dirty_usage) = verified_remote_dirty_usage(&expected_peers, peer_snapshots) else {
|
||||
return default_result(resolution.requested_scope);
|
||||
};
|
||||
dirty_buckets.extend(remote_dirty_buckets);
|
||||
let remote_resolution = resolve_remote_dirty_usage_scope(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
remote_dirty_usage,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
);
|
||||
if !remote_resolution.remote_dirty_usage_acknowledgements.is_empty() {
|
||||
#[cfg(test)]
|
||||
let capability_supported = if let Some(capability_supported) = resolution.test_scoped_dirty_usage_capability {
|
||||
capability_supported
|
||||
} else {
|
||||
let Some(notification_system) = notification_system.as_ref() else {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
};
|
||||
let capability_acknowledgements = remote_resolution
|
||||
.remote_dirty_usage_acknowledgements
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(Into::into)
|
||||
.collect::<Vec<crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement>>();
|
||||
matches!(
|
||||
notification_system
|
||||
.scanner_scoped_dirty_usage_capabilities(capability_acknowledgements)
|
||||
.await,
|
||||
Ok(true)
|
||||
)
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
let capability_supported = {
|
||||
let Some(notification_system) = notification_system.as_ref() else {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
};
|
||||
let capability_acknowledgements = remote_resolution
|
||||
.remote_dirty_usage_acknowledgements
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(Into::into)
|
||||
.collect::<Vec<crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement>>();
|
||||
matches!(
|
||||
notification_system
|
||||
.scanner_scoped_dirty_usage_capabilities(capability_acknowledgements)
|
||||
.await,
|
||||
Ok(true)
|
||||
)
|
||||
};
|
||||
if !capability_supported {
|
||||
return default_result(ScannerBucketScanScope::default());
|
||||
}
|
||||
}
|
||||
return remote_resolution;
|
||||
}
|
||||
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
default_result(scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
|
||||
true,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
)
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn resolve_scanner_bucket_scan_scope_for_tests<S>(
|
||||
store: &S,
|
||||
distributed: bool,
|
||||
resolution: ScannerBucketScopeResolution<'_>,
|
||||
) -> ScannerBucketScopeResolutionResult
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
resolve_scanner_bucket_scan_scope(store, distributed, resolution).await
|
||||
}
|
||||
|
||||
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
|
||||
@@ -183,7 +275,9 @@ where
|
||||
scan_mode,
|
||||
scan_scope,
|
||||
persisted_usage_baseline,
|
||||
observed_usage_candidate,
|
||||
requires_full_scan,
|
||||
service_cohort,
|
||||
#[cfg(test)]
|
||||
resolved_scope_observer,
|
||||
} = request;
|
||||
@@ -275,6 +369,12 @@ where
|
||||
}
|
||||
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
|
||||
bucket_plan_complete &= scanner_bucket_inventory_is_complete(&all_buckets, &buckets_by_source);
|
||||
if bucket_plan_complete && let Some(cohort) = &service_cohort {
|
||||
cohort
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.refresh(&buckets_by_source);
|
||||
}
|
||||
let structural_scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
|
||||
let scan_plan_digest = scanner_bucket_work_digest(structural_scan_plan_digest, scan_mode, requires_full_scan);
|
||||
@@ -282,13 +382,14 @@ where
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
|
||||
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
let scan_scope = resolve_scanner_bucket_scan_scope(
|
||||
let scope_resolution = resolve_scanner_bucket_scan_scope(
|
||||
store,
|
||||
distributed,
|
||||
ScannerBucketScopeResolution {
|
||||
requested_scope: scan_scope,
|
||||
baseline_proof: ScannerCacheBaselineProof {
|
||||
data: persisted_usage_baseline.as_ref(),
|
||||
authoritative_data: persisted_usage_baseline.as_ref(),
|
||||
observed_candidate_data: observed_usage_candidate.as_ref(),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch,
|
||||
want_cycle,
|
||||
@@ -298,9 +399,15 @@ where
|
||||
dirty_usage_snapshot: &dirty_usage_snapshot,
|
||||
all_buckets: &all_buckets,
|
||||
requires_full_scan: requires_full_scan || scan_mode == HealScanMode::Deep,
|
||||
#[cfg(test)]
|
||||
test_peer_snapshots: None,
|
||||
#[cfg(test)]
|
||||
test_scoped_dirty_usage_capability: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
|
||||
let scan_scope = scope_resolution.scope;
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = resolved_scope_observer {
|
||||
let _ = observer.send(scan_scope.clone());
|
||||
@@ -325,12 +432,21 @@ where
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
let empty_usage = DataUsageInfo {
|
||||
last_update: Some(SystemTime::now()),
|
||||
scanner_cycle: Some(want_cycle),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
let Some(candidate) = empty_namespace_usage_candidate(
|
||||
&all_buckets,
|
||||
&expected_sources,
|
||||
&buckets_by_source,
|
||||
ScannerSnapshotIdentity {
|
||||
cycle: want_cycle,
|
||||
leader_epoch,
|
||||
plan_digest: scan_plan_digest,
|
||||
coverage_digest: bucket_coverage_digest,
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
},
|
||||
) else {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
};
|
||||
let (empty_usage, publication_expectation) = candidate.prepare(status);
|
||||
let observational_snapshot_published = if should_publish_observational_snapshot(status) {
|
||||
publish_observational_snapshot(&updates, empty_usage).await?
|
||||
} else {
|
||||
@@ -353,7 +469,8 @@ where
|
||||
.with_activity_digest(activity_digest)
|
||||
.with_observational_snapshot_published(observational_snapshot_published)
|
||||
.with_remote_publication_lease_targets(remote_publication_lease_targets)
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
|
||||
.with_publication_expectation(publication_expectation));
|
||||
}
|
||||
|
||||
let total_results = expected_sources.len();
|
||||
@@ -399,7 +516,32 @@ where
|
||||
let first_err_mutex: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
|
||||
let mut wait_futs = Vec::new();
|
||||
|
||||
for (results_index, set) in set_disks.iter().enumerate() {
|
||||
let set_order = service_cohort.as_ref().map_or_else(
|
||||
|| (0..set_disks.len()).collect::<Vec<_>>(),
|
||||
|cohort| {
|
||||
cohort
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.order_set_indices(&set_disks)
|
||||
},
|
||||
);
|
||||
for results_index in set_order {
|
||||
let set = &set_disks[results_index];
|
||||
// Acquire in dispatch order, not in independently scheduled tasks.
|
||||
// A whole set still shares the existing parent budget; this is not
|
||||
// a per-bucket quantum or a cross-source completion guarantee.
|
||||
let permit_wait_start = Instant::now();
|
||||
let permit = tokio::select! {
|
||||
biased;
|
||||
_ = child_token.cancelled() => break,
|
||||
permit = set_scan_semaphore.clone().acquire_owned() => match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => break,
|
||||
},
|
||||
};
|
||||
if child_token.is_cancelled() || budget.budget_elapsed() {
|
||||
break;
|
||||
}
|
||||
let results_index_clone = results_index;
|
||||
// Clone the Arc to move it into the spawned task
|
||||
let set_clone: Arc<SetDisks> = Arc::clone(set);
|
||||
@@ -414,7 +556,6 @@ where
|
||||
let scan_mode_clone = scan_mode;
|
||||
let results_mutex_clone = results_mutex.clone();
|
||||
let first_err_mutex_clone = first_err_mutex.clone();
|
||||
let set_scan_semaphore_clone = set_scan_semaphore.clone();
|
||||
let queued_set_scans_clone = queued_set_scans.clone();
|
||||
let active_set_scans_clone = active_set_scans.clone();
|
||||
|
||||
@@ -437,6 +578,7 @@ where
|
||||
digest: structural_scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
requires_full_scan,
|
||||
service_cohort: service_cohort.clone(),
|
||||
execution_digest,
|
||||
leader_epoch,
|
||||
tier_registry_generation,
|
||||
@@ -448,15 +590,10 @@ where
|
||||
};
|
||||
// Spawn task to run the scanner
|
||||
let scanner_fut = tokio::spawn(async move {
|
||||
let permit_wait = child_token_clone.clone();
|
||||
let permit_wait_start = Instant::now();
|
||||
let _permit = tokio::select! {
|
||||
permit = set_scan_semaphore_clone.acquire_owned() => match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => return,
|
||||
},
|
||||
_ = permit_wait.cancelled() => return,
|
||||
};
|
||||
let _permit = permit;
|
||||
if child_token_clone.is_cancelled() || budget_clone.budget_elapsed() {
|
||||
return;
|
||||
}
|
||||
metrics::histogram!(
|
||||
METRIC_SCANNER_SET_SCAN_WAIT_SECONDS,
|
||||
"pool" => pool_label.clone(),
|
||||
@@ -562,7 +699,7 @@ where
|
||||
let (activity_status, remote_publication_lease_targets) =
|
||||
scanner_cycle_activity_status(store, distributed, &activity_before).await;
|
||||
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
|
||||
let completed_usage = completed_data_usage_info(
|
||||
let completed_usage = completed_usage_candidate(
|
||||
&results,
|
||||
&ScannerSnapshotScope {
|
||||
sources: &expected_sources,
|
||||
@@ -603,7 +740,10 @@ where
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
let observational_snapshot_published = if let Some((data_usage_info, _)) = completed_usage {
|
||||
let mut publication_expectation = None;
|
||||
let observational_snapshot_published = if let Some(candidate) = completed_usage {
|
||||
let (data_usage_info, expectation) = candidate.prepare(cycle_status);
|
||||
publication_expectation = expectation;
|
||||
if should_publish_observational_snapshot(cycle_status) {
|
||||
publish_observational_snapshot(&updates, data_usage_info).await?
|
||||
} else {
|
||||
@@ -628,11 +768,14 @@ where
|
||||
if cycle_status == ScannerCycleStatus::Complete {
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
}
|
||||
let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let remote_dirty_usage_acknowledgements =
|
||||
if cycle_status == ScannerCycleStatus::Complete && !remote_dirty_usage_acknowledgements.is_empty() {
|
||||
remote_dirty_usage_acknowledgements
|
||||
} else if cycle_status == ScannerCycleStatus::Complete && scan_scope.is_default() {
|
||||
crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_activity_digest(activity_digest)
|
||||
@@ -641,5 +784,6 @@ where
|
||||
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
|
||||
.with_failed_dirty_usage(!failed_buckets.is_empty())
|
||||
.with_pending_maintenance_work(pending_maintenance_work)
|
||||
.with_required_cycle_floor(required_cycle_floor))
|
||||
.with_required_cycle_floor(required_cycle_floor)
|
||||
.with_publication_expectation(publication_expectation))
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ impl ScannerIODisk for Disk {
|
||||
Ok(size_summary)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, budget, updates, cache, set_disks))]
|
||||
#[tracing::instrument(skip(self, budget, updates, cache, set_disks, options), fields(scan_mode = ?options.scan_mode))]
|
||||
async fn nsscanner_disk(
|
||||
self: Arc<Self>,
|
||||
ctx: CancellationToken,
|
||||
@@ -154,8 +154,12 @@ impl ScannerIODisk for Disk {
|
||||
set_disks: Vec<Arc<Disk>>,
|
||||
cache: DataUsageCache,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
scan_mode: HealScanMode,
|
||||
options: ScannerDiskScanOptions,
|
||||
) -> Result<ScannerDiskScanOutcome> {
|
||||
let ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
} = options;
|
||||
let done_drive = Metrics::time(Metric::ScanBucketDrive);
|
||||
let drive_start = std::time::Instant::now();
|
||||
let bucket = cache.info.name.clone();
|
||||
@@ -198,7 +202,19 @@ impl ScannerIODisk for Disk {
|
||||
cache.info.object_lock = Some(Arc::new(object_lock_config));
|
||||
}
|
||||
|
||||
let result = scan_data_folder(
|
||||
// Prefix reuse never crosses semantic maintenance boundaries. A
|
||||
// lifecycle, replication, Object Lock, or erasure health walk can
|
||||
// make a clean data subtree require scanner-side work even without a
|
||||
// direct object mutation in the local journal. The folder scanner
|
||||
// separately rejects scopes in erasure mode.
|
||||
let prefix_scan_scope = (scan_mode == HealScanMode::Normal
|
||||
&& cache.info.lifecycle.is_none()
|
||||
&& cache.info.replication.is_none()
|
||||
&& cache.info.object_lock.is_none())
|
||||
.then_some(prefix_scan_scope)
|
||||
.flatten();
|
||||
|
||||
let result = scan_data_folder_scoped(
|
||||
ctx.clone(),
|
||||
budget,
|
||||
set_disks,
|
||||
@@ -207,6 +223,7 @@ impl ScannerIODisk for Disk {
|
||||
updates,
|
||||
scan_mode,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
prefix_scan_scope,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user