mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 20:46:11 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3340a755c1 | |||
| b4e0838b01 | |||
| 6233466c5d | |||
| 901d052c38 | |||
| fdb3d9f9bb | |||
| 44333e136b | |||
| 9e24d23c30 | |||
| 6277287399 | |||
| 358ff0f0e6 | |||
| 1cea5fa1c4 | |||
| b1a2235cd6 | |||
| e0663c11af | |||
| 11c9fd64ce | |||
| 7f631ec378 | |||
| 3205f85c2a | |||
| ae87ddbe2f | |||
| f09aaad2a9 | |||
| 3ae29aab26 | |||
| 3c54ac1deb | |||
| 44fe0b9950 | |||
| 58f1840630 | |||
| 5e33186cf6 | |||
| ed100103d0 | |||
| 93c89ef132 | |||
| ea4068b8ac |
@@ -582,13 +582,59 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
run: |
|
||||
python3 - <<'PYBUILD'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(["git", *args], text=True).strip()
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with pathlib.Path(path).open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"]
|
||||
commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
|
||||
clean_before = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_before:
|
||||
raise SystemExit("hooks binary requires a clean build checkout")
|
||||
lock_sha256 = sha256("Cargo.lock")
|
||||
lock_git_blob = git("hash-object", "Cargo.lock")
|
||||
rustc = subprocess.check_output(["rustc", "-vV"], text=True)
|
||||
host = next(line.removeprefix("host: ") for line in rustc.splitlines() if line.startswith("host: "))
|
||||
if os.environ.get("CARGO_BUILD_TARGET") or pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target")).resolve() != pathlib.Path("target").resolve():
|
||||
raise SystemExit("this artifact requires the native target/debug output")
|
||||
subprocess.run(argv, check=True)
|
||||
clean_after = not git("status", "--porcelain", "--untracked-files=normal")
|
||||
if not clean_after or commit != git("rev-parse", "HEAD") or tree != git("rev-parse", "HEAD^{tree}") or lock_sha256 != sha256("Cargo.lock"):
|
||||
raise SystemExit("hooks binary source changed while building")
|
||||
manifest = {
|
||||
"schema": 1, "commit": commit, "tree": tree,
|
||||
"clean_before": clean_before, "clean_after": clean_after,
|
||||
"lock_sha256": lock_sha256, "lock_git_blob": lock_git_blob,
|
||||
"argv": argv, "profile": "debug", "target": host,
|
||||
"features": ["e2e-test-hooks"],
|
||||
"rustc_verbose": rustc,
|
||||
"build_flags": {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_TARGET_DIR", "RUSTUP_TOOLCHAIN") if key in os.environ},
|
||||
"binary_sha256": sha256("target/debug/rustfs"),
|
||||
}
|
||||
pathlib.Path("target/debug/rustfs.e2e-startup-cas-build.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
PYBUILD
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug/rustfs
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e-startup-cas-build.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -854,12 +900,6 @@ jobs:
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
# a skip instead of failing, so surface that here where it is visible.
|
||||
if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
|
||||
echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
@@ -912,6 +952,36 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Preserve startup CAS binary input
|
||||
env:
|
||||
STARTUP_CAS_INPUT: ${{ runner.temp }}/rustfs-startup-cas-input
|
||||
run: |
|
||||
python3 - <<'PYINPUT'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
source = pathlib.Path("target/debug/rustfs")
|
||||
manifest_path = source.with_name("rustfs.e2e-startup-cas-build.json")
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
target = pathlib.Path(os.environ["STARTUP_CAS_INPUT"])
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
binary = target / "rustfs"
|
||||
shutil.copy2(source, binary)
|
||||
digest = hashlib.sha256()
|
||||
with binary.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
|
||||
raise SystemExit("downloaded hooks binary identity mismatch")
|
||||
shutil.copy2(manifest_path, target / manifest_path.name)
|
||||
binary.chmod(0o755)
|
||||
PYINPUT
|
||||
|
||||
- name: Verify e2e full membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
@@ -924,6 +994,10 @@ jobs:
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
run: cargo nextest run --profile e2e-full -p e2e_test
|
||||
|
||||
- name: Upload junit
|
||||
@@ -936,6 +1010,17 @@ jobs:
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: fresh-startup-cas-evidence-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
name: End-to-End Tests (rio-v2)
|
||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
||||
|
||||
@@ -121,22 +121,6 @@ jobs:
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
# Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
|
||||
# prerelease (alpha/beta/rc) also moves `latest`, so users pulling
|
||||
# `latest` get the newest test build. Once a stable tag is published
|
||||
# this returns false and `latest` follows stable releases only.
|
||||
prerelease_moves_latest() {
|
||||
local stable_tags
|
||||
stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
|
||||
| awk '{print $2}' \
|
||||
| grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
|
||||
if [[ -z "$stable_tags" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "ℹ️ Stable release tag(s) already exist; prereleases no longer update latest"
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
echo "🔗 Triggered by build workflow completion"
|
||||
@@ -200,8 +184,8 @@ jobs:
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
create_latest=true
|
||||
echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
|
||||
else
|
||||
@@ -259,8 +243,8 @@ jobs:
|
||||
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
|
||||
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
|
||||
create_latest=true
|
||||
echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
|
||||
else
|
||||
@@ -410,13 +394,11 @@ jobs:
|
||||
TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
|
||||
TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
|
||||
|
||||
# Add latest when requested (stable releases, and prereleases before GA)
|
||||
# Add channel tags for prereleases and latest for stable
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
|
||||
fi
|
||||
|
||||
# Always add the channel tag for prereleases, independent of latest
|
||||
if [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
# Prerelease channel tags (alpha, beta, rc)
|
||||
if [[ "$VERSION" == *"alpha"* ]]; then
|
||||
CHANNEL="alpha"
|
||||
@@ -573,7 +555,7 @@ jobs:
|
||||
"prerelease")
|
||||
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
|
||||
echo "⚠️ This is a prerelease image - use with caution"
|
||||
# Prereleases move latest until the first stable tag exists (pre-GA policy).
|
||||
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
echo "🏷️ Latest tag has been created for prerelease: $VERSION"
|
||||
else
|
||||
|
||||
Generated
+28
-29
@@ -2527,18 +2527,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.17"
|
||||
version = "0.5.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
|
||||
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.8"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
||||
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
@@ -2546,27 +2546,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.21"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.14"
|
||||
version = "0.3.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
|
||||
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.23"
|
||||
version = "0.8.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
||||
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
@@ -3673,9 +3673,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.2"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"pem-rfc7468 1.0.0",
|
||||
@@ -4057,6 +4057,7 @@ dependencies = [
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"suppaftp",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -4090,7 +4091,7 @@ version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve 0.14.1",
|
||||
"rfc6979 0.6.0",
|
||||
@@ -5734,9 +5735,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.12.2"
|
||||
version = "2.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
|
||||
checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -6139,9 +6140,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
version = "2.3.2"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "561a8da1a50e1428d3c51321dafeca849df992a5bb67720c386131234caba82e"
|
||||
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"crc32fast",
|
||||
@@ -7933,7 +7934,7 @@ version = "0.8.0-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"spki 0.8.0",
|
||||
]
|
||||
|
||||
@@ -7976,7 +7977,7 @@ dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand_core 0.10.1",
|
||||
"scrypt 0.12.0",
|
||||
@@ -8000,7 +8001,7 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"pkcs5 0.8.1",
|
||||
"rand_core 0.10.1",
|
||||
"spki 0.8.0",
|
||||
@@ -8942,9 +8943,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.7.0"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
|
||||
checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -9326,7 +9327,7 @@ dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"digest 0.11.3",
|
||||
"ecdsa 0.17.0",
|
||||
"ed25519-dalek 3.0.0",
|
||||
@@ -9999,7 +10000,6 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"serial_test",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
@@ -10746,7 +10746,6 @@ dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
"rustfs-lock",
|
||||
@@ -10944,9 +10943,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-uring"
|
||||
version = "0.2.2"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b29bc57b4bd62a73f4fae408b536adf578332e50e464797d09dc2382c7cb68c2"
|
||||
checksum = "0486e62d0efe25db95c00aeacb2da84368adcba299216cda99fcb11328061c84"
|
||||
dependencies = [
|
||||
"io-uring",
|
||||
"libc",
|
||||
@@ -11421,7 +11420,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"ctutils",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11995,7 +11994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.2",
|
||||
"der 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+5
-6
@@ -191,7 +191,6 @@ rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
@@ -257,10 +256,10 @@ clap = { version = "4.6.6" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, version = "55.0.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
@@ -307,7 +306,7 @@ rustfs-erasure-codec = { version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
|
||||
@@ -422,9 +422,9 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Legacy, unverified repair notice. Its identity lacks kind, set scope,
|
||||
/// bucket incarnation and responsibility generation. Consumers must not use
|
||||
/// it to discharge persisted repair responsibility.
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
@@ -439,8 +439,8 @@ const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record a legacy notification for compatibility. This is not an
|
||||
/// acknowledgement of storage verification or durable repair completion.
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
@@ -515,9 +515,6 @@ mod tests {
|
||||
}
|
||||
coalescer_release(&key, Some(lease));
|
||||
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
|
||||
assert_ne!(lease, retry_lease);
|
||||
coalescer_release(&key, Some(lease));
|
||||
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
|
||||
coalescer_release(&key, Some(retry_lease));
|
||||
}
|
||||
|
||||
|
||||
@@ -130,37 +130,6 @@ Scanner cycle budget controls:
|
||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
||||
|
||||
## Foreground write admission environment variables
|
||||
|
||||
Large direct `PutObject` requests and multipart `UploadPart` requests share one
|
||||
per-process permit pool that bounds how many bodies are ingested and written
|
||||
concurrently. Small direct PUTs stay on the legacy path.
|
||||
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE`
|
||||
- enables the default-on pool; `false` keeps only the soft request counter.
|
||||
- default is `true`.
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT`
|
||||
- permits in the pool; `0` derives half of `RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS`, clamped to `32`.
|
||||
- default is `0` (32 permits at stock settings).
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES`
|
||||
- smallest direct `PutObject` that takes a permit; unknown-size requests always do.
|
||||
- default is `33554432` (32 MiB).
|
||||
- `RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- how long a direct `PutObject` waits for a permit before returning S3 `SlowDown`.
|
||||
- default is `250`.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES`
|
||||
- smallest `UploadPart` that takes a permit; `0` gates every part.
|
||||
- default is `0`.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- how long an `UploadPart` waits in the bounded queue for a permit before returning S3 `SlowDown`; `0` rejects immediately when the pool is full.
|
||||
- default is `30000`. Parts wait before body ingest, so SDK-default clients that send every part of an upload concurrently drain through the pool instead of failing.
|
||||
- `RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING`
|
||||
- maximum `UploadPart` requests waiting for a permit at once; parts beyond it return `SlowDown` without waiting.
|
||||
- default is `0`, which derives 16 times the permit limit (512 at stock settings).
|
||||
- `RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE`, `RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT`, `RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`
|
||||
- experimental strict gate that applies to every foreground write regardless of size and replaces the pool above when enabled.
|
||||
- default is disabled; enabling it with limit `0` disables foreground write admission entirely.
|
||||
|
||||
## Remote tier timeout environment variables
|
||||
|
||||
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
|
||||
|
||||
@@ -365,36 +365,13 @@ pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str =
|
||||
"RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 0;
|
||||
|
||||
/// Time in milliseconds an automatic foreground direct PutObject waits for a permit.
|
||||
/// Time in milliseconds an automatic foreground write waits for a permit.
|
||||
///
|
||||
/// A short wait smooths transient bursts while still returning S3
|
||||
/// `SlowDown`/503 before body ingest when the node is already saturated.
|
||||
pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 250;
|
||||
|
||||
/// Time in milliseconds a multipart UploadPart waits for a foreground write permit.
|
||||
///
|
||||
/// SDK-default multipart clients send every part of an upload concurrently, so
|
||||
/// a single node routinely sees several times more parts in flight than the
|
||||
/// permit pool allows. Those parts have not ingested a body yet, so queueing
|
||||
/// them costs a connection rather than memory or internode streams; the pool
|
||||
/// still bounds the number of parts being written. The wait is long enough for
|
||||
/// an ordinary queue to drain on modest hardware, and a part that cannot get a
|
||||
/// permit within it fails with S3 `SlowDown`/503 for the client to retry.
|
||||
/// `0` rejects immediately when the pool is full.
|
||||
pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str =
|
||||
"RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 30_000;
|
||||
|
||||
/// Maximum multipart UploadPart requests waiting for a foreground write permit per process.
|
||||
///
|
||||
/// Parts beyond this queue depth are rejected with S3 `SlowDown`/503 without
|
||||
/// waiting, so a genuinely saturated node still fails fast instead of holding
|
||||
/// an unbounded set of connections open for the whole wait timeout.
|
||||
/// `0` derives the depth from the permit limit.
|
||||
pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING: &str = "RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MAX_PENDING: usize = 0;
|
||||
|
||||
const _: () = assert!(DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE);
|
||||
|
||||
/// Environment variable for minimum GetObject timeout in seconds.
|
||||
|
||||
@@ -144,3 +144,6 @@ russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -184,8 +184,6 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,14 +34,12 @@ use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CommonPrefix, CompleteMultipartUploadInput,
|
||||
CompleteMultipartUploadOutput, CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput,
|
||||
DeleteObjectOutput, DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput,
|
||||
GetBucketVersioningOutput, GetObjectInput, GetObjectLegalHoldInput, GetObjectLegalHoldOutput,
|
||||
GetObjectLockConfigurationInput, GetObjectLockConfigurationOutput, GetObjectOutput, GetObjectRetentionInput,
|
||||
GetObjectRetentionOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput, HeadObjectInput,
|
||||
GetBucketVersioningOutput, GetObjectInput, GetObjectLockConfigurationInput, GetObjectLockConfigurationOutput,
|
||||
GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput, HeadObjectInput,
|
||||
HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsV2Input, ListObjectsV2Output, Object,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectStorageClass, ObjectVersionId, PutObjectInput, PutObjectLegalHoldInput,
|
||||
PutObjectLegalHoldOutput, PutObjectOutput, PutObjectRetentionInput, PutObjectRetentionOutput, PutObjectTaggingInput,
|
||||
PutObjectTaggingOutput, Range, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectStorageClass, ObjectVersionId, PutObjectInput, PutObjectOutput,
|
||||
PutObjectTaggingInput, PutObjectTaggingOutput, Range, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -129,10 +127,6 @@ pub enum Operation {
|
||||
GetObjectTagging,
|
||||
PutObjectTagging,
|
||||
DeleteObjectTagging,
|
||||
GetObjectRetention,
|
||||
PutObjectRetention,
|
||||
GetObjectLegalHold,
|
||||
PutObjectLegalHold,
|
||||
ListObjectVersions,
|
||||
ListObjectsV2,
|
||||
CreateMultipartUpload,
|
||||
@@ -507,10 +501,6 @@ struct StoreState {
|
||||
/// PutObject carrying any `x-amz-object-lock-*` header must also carry
|
||||
/// `Content-MD5` or an `x-amz-checksum-*` header.
|
||||
require_checksum_for_object_lock: bool,
|
||||
/// Models Wasabi (rustfs/backlog#2340): a version-addressed DELETE of a
|
||||
/// version id the target never had answers 404 `NoSuchVersion` instead of
|
||||
/// the idempotent 204 RustFS/MinIO give.
|
||||
reject_unknown_version_deletes: bool,
|
||||
limits: StoreLimits,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
@@ -575,41 +565,6 @@ struct ObjectVersion {
|
||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||
/// target behavior); empty when the drop mode discarded them.
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
/// Object Lock state of the version: retention (mode, retain-until) from
|
||||
/// the PUT / CreateMultipartUpload headers or PutObjectRetention, and the
|
||||
/// legal hold flag; replayed on HEAD.
|
||||
lock: VersionLock,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct VersionLock {
|
||||
retention: Option<(String, Timestamp)>,
|
||||
/// `None` until a legal hold status was ever set; like S3, HEAD then
|
||||
/// reports nothing, while an explicit OFF is reported as `OFF`.
|
||||
legal_hold: Option<bool>,
|
||||
}
|
||||
|
||||
impl VersionLock {
|
||||
fn from_headers(
|
||||
mode: Option<ObjectLockMode>,
|
||||
retain_until: Option<Timestamp>,
|
||||
legal_hold: Option<ObjectLockLegalHoldStatus>,
|
||||
) -> Self {
|
||||
Self {
|
||||
retention: mode.zip(retain_until).map(|(mode, until)| (mode.as_str().to_string(), until)),
|
||||
legal_hold: legal_hold.map(|status| status.as_str().eq_ignore_ascii_case("ON")),
|
||||
}
|
||||
}
|
||||
|
||||
fn legal_hold_status(&self) -> Option<ObjectLockLegalHoldStatus> {
|
||||
self.legal_hold.map(|on| {
|
||||
ObjectLockLegalHoldStatus::from_static(if on {
|
||||
ObjectLockLegalHoldStatus::ON
|
||||
} else {
|
||||
ObjectLockLegalHoldStatus::OFF
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -621,7 +576,6 @@ struct MultipartState {
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
standard_headers: StandardHeaders,
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
lock: VersionLock,
|
||||
parts: BTreeMap<i32, MultipartPart>,
|
||||
}
|
||||
|
||||
@@ -891,7 +845,6 @@ impl FakeS3Target {
|
||||
standard_headers: seed.standard_headers.clone(),
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
};
|
||||
upsert_version(&mut state, bucket, key.into(), version).expect("seed object must fit the storage budget");
|
||||
e_tag
|
||||
@@ -965,12 +918,6 @@ impl FakeS3Target {
|
||||
/// PutObject that carries Object Lock parameters (AWS S3 / MinIO rule,
|
||||
/// rustfs#7082). `Content-MD5`, when present, is always verified against
|
||||
/// the body regardless of this mode.
|
||||
/// Wasabi-like mode: DELETE of an unknown version id answers 404
|
||||
/// `NoSuchVersion` (the default 204 models RustFS/MinIO).
|
||||
pub fn reject_unknown_version_deletes(&self, enabled: bool) {
|
||||
lock(&self.backend.store).reject_unknown_version_deletes = enabled;
|
||||
}
|
||||
|
||||
pub fn require_checksum_for_object_lock(&self, enabled: bool) {
|
||||
lock(&self.backend.store).require_checksum_for_object_lock = enabled;
|
||||
}
|
||||
@@ -1205,10 +1152,6 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||
"GetObjectRetention" => Operation::GetObjectRetention,
|
||||
"PutObjectRetention" => Operation::PutObjectRetention,
|
||||
"GetObjectLegalHold" => Operation::GetObjectLegalHold,
|
||||
"PutObjectLegalHold" => Operation::PutObjectLegalHold,
|
||||
"ListObjectsV2" => Operation::ListObjectsV2,
|
||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||
"UploadPart" => Operation::UploadPart,
|
||||
@@ -1346,18 +1289,6 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::DeleteObjectTagging
|
||||
}
|
||||
(&Method::GET, true) if query.contains_key("retention") && only_query_keys(&["retention", "versionId"]) => {
|
||||
Operation::GetObjectRetention
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("retention") && only_query_keys(&["retention", "versionId"]) => {
|
||||
Operation::PutObjectRetention
|
||||
}
|
||||
(&Method::GET, true) if query.contains_key("legal-hold") && only_query_keys(&["legal-hold", "versionId"]) => {
|
||||
Operation::GetObjectLegalHold
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("legal-hold") && only_query_keys(&["legal-hold", "versionId"]) => {
|
||||
Operation::PutObjectLegalHold
|
||||
}
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
@@ -1911,28 +1842,6 @@ fn set_version_tags(
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
fn update_version_lock(
|
||||
state: &mut StoreState,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
update: impl FnOnce(&mut VersionLock),
|
||||
) -> S3Result<String> {
|
||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||
let version = state
|
||||
.buckets
|
||||
.get_mut(bucket)
|
||||
.expect("bucket existence checked by find_version")
|
||||
.objects
|
||||
.get_mut(key)
|
||||
.expect("key existence checked by find_version")
|
||||
.iter_mut()
|
||||
.find(|version| version.version_id == resolved)
|
||||
.expect("version existence checked by find_version");
|
||||
update(&mut version.lock);
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
/// Whether version ids are surfaced for this bucket. Unknown buckets report
|
||||
/// `true`; the caller's lookup raises `NoSuchBucket` first.
|
||||
fn bucket_versioned(state: &StoreState, bucket: &str) -> bool {
|
||||
@@ -2372,11 +2281,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
lock: VersionLock::from_headers(
|
||||
input.object_lock_mode,
|
||||
input.object_lock_retain_until_date,
|
||||
input.object_lock_legal_hold_status,
|
||||
),
|
||||
};
|
||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -2435,13 +2339,6 @@ impl S3 for FakeBackend {
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: versioned.then_some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
object_lock_mode: version
|
||||
.lock
|
||||
.retention
|
||||
.as_ref()
|
||||
.map(|(mode, _)| ObjectLockMode::from(mode.clone())),
|
||||
object_lock_retain_until_date: version.lock.retention.as_ref().map(|(_, until)| until.clone()),
|
||||
object_lock_legal_hold_status: version.lock.legal_hold_status(),
|
||||
..Default::default()
|
||||
});
|
||||
response.status = served.status;
|
||||
@@ -2476,13 +2373,6 @@ impl S3 for FakeBackend {
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: versioned.then_some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
object_lock_mode: version
|
||||
.lock
|
||||
.retention
|
||||
.as_ref()
|
||||
.map(|(mode, _)| ObjectLockMode::from(mode.clone())),
|
||||
object_lock_retain_until_date: version.lock.retention.as_ref().map(|(_, until)| until.clone()),
|
||||
object_lock_legal_hold_status: version.lock.legal_hold_status(),
|
||||
..Default::default()
|
||||
});
|
||||
response.status = served.status;
|
||||
@@ -2542,82 +2432,6 @@ impl S3 for FakeBackend {
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_retention(
|
||||
&self,
|
||||
req: S3Request<GetObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<GetObjectRetentionOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = find_version(&lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref())?;
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectRetentionOutput {
|
||||
retention: version.lock.retention.map(|(mode, until)| ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from(mode)),
|
||||
retain_until_date: Some(until),
|
||||
}),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_retention(
|
||||
&self,
|
||||
req: S3Request<PutObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<PutObjectRetentionOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let retention = input
|
||||
.retention
|
||||
.and_then(|retention| retention.mode.zip(retention.retain_until_date))
|
||||
.map(|(mode, until)| (mode.as_str().to_string(), until));
|
||||
update_version_lock(&mut lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref(), |lock| {
|
||||
lock.retention = retention;
|
||||
})?;
|
||||
Ok(apply_response_fault(S3Response::new(PutObjectRetentionOutput::default()), fault.as_ref()))
|
||||
}
|
||||
|
||||
async fn get_object_legal_hold(
|
||||
&self,
|
||||
req: S3Request<GetObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<GetObjectLegalHoldOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = find_version(&lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref())?;
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectLegalHoldOutput {
|
||||
legal_hold: Some(ObjectLockLegalHold {
|
||||
status: Some(
|
||||
version
|
||||
.lock
|
||||
.legal_hold_status()
|
||||
.unwrap_or_else(|| ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_legal_hold(
|
||||
&self,
|
||||
req: S3Request<PutObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<PutObjectLegalHoldOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let legal_hold_on = input
|
||||
.legal_hold
|
||||
.and_then(|hold| hold.status)
|
||||
.is_some_and(|status| status.as_str().eq_ignore_ascii_case("ON"));
|
||||
update_version_lock(&mut lock(&self.store), &input.bucket, &input.key, input.version_id.as_deref(), |lock| {
|
||||
lock.legal_hold = Some(legal_hold_on);
|
||||
})?;
|
||||
Ok(apply_response_fault(S3Response::new(PutObjectLegalHoldOutput::default()), fault.as_ref()))
|
||||
}
|
||||
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
@@ -2671,7 +2485,6 @@ impl S3 for FakeBackend {
|
||||
return Ok(apply_response_fault(S3Response::new(DeleteObjectOutput::default()), fault.as_ref()));
|
||||
}
|
||||
if let Some(version_id) = input.version_id {
|
||||
let reject_unknown = state.reject_unknown_version_deletes;
|
||||
let (removed_bytes, removed_versions, delete_marker, remove_key) = {
|
||||
let Some(versions) = state
|
||||
.buckets
|
||||
@@ -2680,9 +2493,6 @@ impl S3 for FakeBackend {
|
||||
.objects
|
||||
.get_mut(&input.key)
|
||||
else {
|
||||
if reject_unknown {
|
||||
return Err(s3s::s3_error!(NoSuchVersion, "The specified version does not exist."));
|
||||
}
|
||||
return Ok(apply_response_fault(
|
||||
S3Response::new(DeleteObjectOutput {
|
||||
version_id: Some(version_id),
|
||||
@@ -2691,9 +2501,6 @@ impl S3 for FakeBackend {
|
||||
fault.as_ref(),
|
||||
));
|
||||
};
|
||||
if reject_unknown && !versions.iter().any(|version| version.version_id == version_id) {
|
||||
return Err(s3s::s3_error!(NoSuchVersion, "The specified version does not exist."));
|
||||
}
|
||||
let mut removed_bytes = 0usize;
|
||||
let mut removed_versions = 0usize;
|
||||
let mut delete_marker = None;
|
||||
@@ -2747,7 +2554,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers: StandardHeaders::default(),
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
},
|
||||
)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -2802,11 +2608,6 @@ impl S3 for FakeBackend {
|
||||
metadata: input.metadata,
|
||||
standard_headers,
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
lock: VersionLock::from_headers(
|
||||
input.object_lock_mode,
|
||||
input.object_lock_retain_until_date,
|
||||
input.object_lock_legal_hold_status,
|
||||
),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -2947,7 +2748,6 @@ impl S3 for FakeBackend {
|
||||
metadata: upload.metadata.clone(),
|
||||
standard_headers: upload.standard_headers.clone(),
|
||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||
lock: upload.lock.clone(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
selected,
|
||||
@@ -2978,7 +2778,6 @@ impl S3 for FakeBackend {
|
||||
standard_headers: upload.standard_headers,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: upload.replication_sse_headers,
|
||||
lock: upload.lock,
|
||||
};
|
||||
let mut state = lock(&self.store);
|
||||
let versioned = bucket_versioned(&state, &input.bucket);
|
||||
@@ -4810,7 +4609,6 @@ mod tests {
|
||||
metadata: None,
|
||||
standard_headers: StandardHeaders::default(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
lock: VersionLock::default(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -34,8 +34,6 @@ mod tests {
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::info;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tracing::warn;
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
@@ -54,23 +52,6 @@ mod tests {
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerHealEvidenceCase {
|
||||
id: &'static str,
|
||||
oracle: &'static str,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart",
|
||||
oracle: "background-target-restart.json",
|
||||
};
|
||||
|
||||
struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
@@ -85,22 +66,10 @@ mod tests {
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
@@ -120,10 +89,10 @@ mod tests {
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
if directory.join("background-target-restart.json").exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
Ok(Some((directory, run)))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
@@ -146,49 +115,6 @@ mod tests {
|
||||
}
|
||||
|
||||
impl TcpPortBlackhole {
|
||||
/// Environment flag that turns an unusable fault-injection host into a
|
||||
/// hard failure instead of a logged skip. Lanes that provision
|
||||
/// `CAP_NET_ADMIN` set it so a broken runner cannot pass silently.
|
||||
#[cfg(target_os = "linux")]
|
||||
const REQUIRE_ENV: &str = "RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION";
|
||||
|
||||
/// Probe whether this host can manipulate the OUTPUT chain at all.
|
||||
///
|
||||
/// Returns `Ok(Some(reason))` when `iptables` is missing or lacks
|
||||
/// `CAP_NET_ADMIN` (the nf_tables backend reports "Permission denied"
|
||||
/// even under `sudo` inside an unprivileged container) and the lane did
|
||||
/// not demand fault injection; returns an error when the lane demands
|
||||
/// it; returns `Ok(None)` when the blackhole can be installed.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn unavailable_reason() -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
|
||||
let id = Command::new("id").arg("-u").output()?;
|
||||
if !id.status.success() {
|
||||
return Err(format!("failed to determine the test process uid: {}", String::from_utf8_lossy(&id.stderr)).into());
|
||||
}
|
||||
let use_sudo = String::from_utf8_lossy(&id.stdout).trim() != "0";
|
||||
let mut command = if use_sudo {
|
||||
let mut command = Command::new("sudo");
|
||||
command.args(["-n", "iptables"]);
|
||||
command
|
||||
} else {
|
||||
Command::new("iptables")
|
||||
};
|
||||
let probe = command.args(["-w", "5", "-S", "OUTPUT"]).output();
|
||||
let reason = match probe {
|
||||
Ok(output) if output.status.success() => return Ok(None),
|
||||
Ok(output) => format!(
|
||||
"iptables cannot read the OUTPUT chain (status {}): {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
),
|
||||
Err(err) => format!("iptables is not runnable: {err}"),
|
||||
};
|
||||
if std::env::var_os(Self::REQUIRE_ENV).is_some() {
|
||||
return Err(format!("{} is set but network fault injection is unavailable: {reason}", Self::REQUIRE_ENV).into());
|
||||
}
|
||||
Ok(Some(reason))
|
||||
}
|
||||
|
||||
fn install(address: &str) -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
let address = address.parse::<SocketAddr>()?;
|
||||
if !address.ip().is_loopback() {
|
||||
@@ -273,27 +199,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a disk directory underneath a running server. Background writers
|
||||
/// (scanner, usage cache, heal markers) can recreate entries between the
|
||||
/// recursive listing and the final `rmdir`, which surfaces as
|
||||
/// `DirectoryNotEmpty` on macOS; retry briefly so the wipe reflects the
|
||||
/// operator action rather than a listing race.
|
||||
fn wipe_directory_while_server_runs(disk: &Path) -> std::io::Result<()> {
|
||||
let mut last_err = None;
|
||||
for _ in 0..20 {
|
||||
match std::fs::remove_dir_all(disk) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::DirectoryNotEmpty => {
|
||||
last_err = Some(err);
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(last_err.expect("retry loop only exits without success after recording an error"))
|
||||
}
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(path) else {
|
||||
return false;
|
||||
@@ -576,7 +481,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
wipe_directory_while_server_runs(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
std::fs::remove_dir_all(&disk0).expect("disk0 wipe should succeed while server is running");
|
||||
std::fs::create_dir_all(&disk0).expect("disk0 should be recreated empty while server is running");
|
||||
assert!(!has_file_under(&disk0), "disk0 must be empty immediately after runtime wipe");
|
||||
|
||||
@@ -963,18 +868,6 @@ mod tests {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_after_target_endpoint_blackhole() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
if let Some(reason) = TcpPortBlackhole::unavailable_reason()? {
|
||||
init_logging();
|
||||
warn!(
|
||||
event = "heal_interruption_skipped",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind = "target_endpoint_blackhole",
|
||||
reason,
|
||||
"Skipping endpoint blackhole scenario: network fault injection is unavailable on this host"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::TargetEndpointBlackhole),
|
||||
@@ -993,7 +886,7 @@ mod tests {
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
restart_evidence_run(&server_binary)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -1642,20 +1535,15 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some(evidence_context) = evidence_run {
|
||||
if let Some((directory, run)) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(
|
||||
file_sha256(&server_binary)?,
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": "process-restart",
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
|
||||
"run_id": run.run_id, "source_revision": run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
@@ -1667,7 +1555,7 @@ mod tests {
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(evidence_context.directory.join(evidence_context.case.oracle))?;
|
||||
.open(directory.join("background-target-restart.json"))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use super::common::{BoxError, OdmSourceSpec, OdmTestEnv, SeedObject};
|
||||
use crate::fake_s3_target::{BucketMode, Operation};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, ObjectAttributes, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -103,19 +103,11 @@ async fn get_miss_pulls_inline_and_serves_locally_afterwards() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_large_object_streams_through_and_backfills_in_background() -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let bucket = "odm-get-large";
|
||||
let env = configured_env(bucket, |spec| {
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = u64::try_from(PART_SIZE).expect("part size fits in u64");
|
||||
})
|
||||
.await?;
|
||||
let env = configured_env(bucket, |spec| spec.policy.inline_max_bytes = 4096).await?;
|
||||
let key = "large/archive.bin";
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let etag = env
|
||||
.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())])
|
||||
.remove(0);
|
||||
assert_eq!(etag.len(), 32, "the source fixture has a plain MD5 ETag");
|
||||
let body = payload(512 * 1024);
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(key, body.clone())]);
|
||||
|
||||
let response = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(response.status, 200, "{}", String::from_utf8_lossy(&response.body));
|
||||
@@ -133,68 +125,6 @@ async fn get_large_object_streams_through_and_backfills_in_background() -> TestR
|
||||
vec![None, None],
|
||||
"one passthrough GET plus one background pull, both unranged"
|
||||
);
|
||||
|
||||
let source_requests = env.source.requests().len();
|
||||
let second_part = env.client.get_object().bucket(bucket).key(key).part_number(2).send().await?;
|
||||
assert_eq!(second_part.content_length(), Some(4096), "the completed second part is the tail");
|
||||
assert_eq!(
|
||||
second_part.content_range(),
|
||||
Some(format!("bytes {PART_SIZE}-{}/{}", body.len() - 1, body.len()).as_str()),
|
||||
"partNumber reads the stored multipart boundary"
|
||||
);
|
||||
assert_eq!(
|
||||
second_part.body.collect().await?.into_bytes(),
|
||||
body.slice(PART_SIZE..),
|
||||
"the local second part contains the exact source tail"
|
||||
);
|
||||
let third_part = env
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.part_number(3)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the completed object has exactly two parts");
|
||||
assert_eq!(third_part.code(), Some("InvalidPart"));
|
||||
|
||||
let mut part_marker = None;
|
||||
for (part_number, part_size) in [(1, PART_SIZE), (2, 4096)] {
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.max_parts(1)
|
||||
.set_part_number_marker(part_marker.clone())
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|value| value.trim_matches('"')),
|
||||
Some(etag.as_str()),
|
||||
"multipart write-back preserves the source MD5 ETag"
|
||||
);
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.expect("RustFS must expose the stored multipart layout");
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(parts.max_parts(), Some(1));
|
||||
assert_eq!(parts.is_truncated(), Some(part_number == 1));
|
||||
assert_eq!(parts.parts().len(), 1, "RustFS returns one stored part per requested page");
|
||||
assert_eq!(parts.parts()[0].part_number(), Some(part_number));
|
||||
assert_eq!(parts.parts()[0].size(), Some(i64::try_from(part_size).expect("part size fits in i64")));
|
||||
part_marker = parts.next_part_number_marker().map(str::to_owned);
|
||||
if part_number == 1 {
|
||||
assert_eq!(part_marker.as_deref(), Some("1"), "the next request continues after the first part");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
env.source.requests().len(),
|
||||
source_requests,
|
||||
"local part reads must not consult the source"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -23,20 +23,16 @@
|
||||
|
||||
use super::common::{
|
||||
ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with, start_source_rustfs,
|
||||
OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
|
||||
use crate::object_lock::common::put_object_lock_configuration;
|
||||
use crate::replication_extension_test::{
|
||||
ReplicationTargetOptions, enable_bucket_versioning, set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
ObjectAttributes, ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use local_ip_address::local_ip;
|
||||
@@ -584,130 +580,6 @@ async fn test_odm_pulled_object_replicates_and_target_as_source_is_rejected() ->
|
||||
"a bucket may not migrate from its own replication target: {}",
|
||||
rejected.body
|
||||
);
|
||||
Box::pin(assert_odm_multipart_replicates_to_rustfs(&env, bucket)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_odm_multipart_replicates_to_rustfs(env: &OdmTestEnv, bucket: &str) -> TestResult {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let replica = start_source_rustfs().await?;
|
||||
let replica_bucket = "odm-real-replica";
|
||||
replica.create_test_bucket(replica_bucket).await?;
|
||||
enable_bucket_versioning(&replica, replica_bucket).await?;
|
||||
let arn = set_replication_target_with_options(
|
||||
&env.rustfs,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &replica.address,
|
||||
access_key: &replica.access_key,
|
||||
secret_key: &replica.secret_key,
|
||||
target_bucket: replica_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&env.rustfs, bucket, &arn).await?;
|
||||
let mut spec = env.fake_source_spec(SOURCE_BUCKET);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
let key = "replicated/preserved-md5-multipart.bin";
|
||||
let body = payload(PART_SIZE + 4096);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(etag.len(), 32, "the source must retain a single-PUT MD5 ETag");
|
||||
assert!(etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(env.wait_local_listed(bucket, key, SETTLE).await?, "the multipart pull must persist");
|
||||
|
||||
let deadline = Instant::now() + SETTLE;
|
||||
let source_head = loop {
|
||||
let head = env.client.head_object().bucket(bucket).key(key).send().await?;
|
||||
match head.replication_status().map(|status| status.as_str()) {
|
||||
Some("COMPLETED") => break head,
|
||||
Some("FAILED") => return Err("the ODM multipart copy failed replication to RustFS".into()),
|
||||
_ => {
|
||||
assert!(Instant::now() < deadline, "the ODM multipart copy never completed replication to RustFS");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
let version = source_head
|
||||
.version_id()
|
||||
.ok_or("the versioned ODM copy omitted its version id")?;
|
||||
assert_ne!(version, "null");
|
||||
let replica_client = replica.create_s3_client();
|
||||
for (client, object_bucket) in [(&env.client, bucket), (&replica_client, replica_bucket)] {
|
||||
let attributes = client
|
||||
.get_object_attributes()
|
||||
.bucket(object_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|value| value.trim_matches('"')), Some(etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the local copy and replica must both expose two parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
}
|
||||
// REPLICA status surfaces on HEAD, like the other inbound-replica checks.
|
||||
let replica_head = replica_client
|
||||
.head_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_get = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica_get.version_id(), Some(version));
|
||||
assert_eq!(replica_get.body.collect().await?.into_bytes(), body);
|
||||
let boundary = replica_client
|
||||
.get_object()
|
||||
.bucket(replica_bucket)
|
||||
.key(key)
|
||||
.version_id(version)
|
||||
.range(format!("bytes={}-{}", PART_SIZE - 32, PART_SIZE + 31))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(boundary.body.collect().await?.into_bytes(), body.slice(PART_SIZE - 32..PART_SIZE + 32));
|
||||
assert_eq!(
|
||||
env.source.count_requests(Operation::GetObject, key),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication and local reads must not fetch the migration source again"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -368,23 +368,23 @@ impl Drop for SlowReplicationTargetGuard {
|
||||
// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
|
||||
// shape `mc replicate resync status` decodes.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct ReplicationResetStatusResponse {
|
||||
struct ReplicationResetStatusResponse {
|
||||
#[serde(rename = "target", default)]
|
||||
pub(crate) targets: Vec<ReplicationResetStatusTarget>,
|
||||
targets: Vec<ReplicationResetStatusTarget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct ReplicationResetStatusTarget {
|
||||
struct ReplicationResetStatusTarget {
|
||||
#[serde(rename = "arn", default)]
|
||||
pub(crate) arn: String,
|
||||
arn: String,
|
||||
#[serde(rename = "resetid", default)]
|
||||
pub(crate) reset_id: String,
|
||||
reset_id: String,
|
||||
#[serde(rename = "resyncStatus", default)]
|
||||
pub(crate) status: String,
|
||||
status: String,
|
||||
#[serde(rename = "replicationCount", default)]
|
||||
pub(crate) replicated_count: i64,
|
||||
replicated_count: i64,
|
||||
#[serde(rename = "object", default)]
|
||||
pub(crate) object: String,
|
||||
object: String,
|
||||
}
|
||||
|
||||
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
@@ -512,7 +512,7 @@ pub(crate) async fn put_bucket_replication(
|
||||
put_bucket_replication_with_delete_statuses(env, bucket, target_arn, "Enabled", None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_bucket_replication_with_delete_statuses(
|
||||
async fn put_bucket_replication_with_delete_statuses(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
target_arn: &str,
|
||||
@@ -627,7 +627,7 @@ async fn put_bucket_replication_rules(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_bucket_replication(
|
||||
async fn delete_bucket_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
@@ -2294,7 +2294,7 @@ async fn site_replication_state_edit(
|
||||
/// return the target `(arn, reset_id)`, asserting the response carries the
|
||||
/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
|
||||
/// that `mc replicate resync start` decodes.
|
||||
pub(crate) async fn start_bucket_replication_reset(
|
||||
async fn start_bucket_replication_reset(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
@@ -2314,7 +2314,7 @@ pub(crate) async fn start_bucket_replication_reset(
|
||||
Ok((arn, reset_id))
|
||||
}
|
||||
|
||||
pub(crate) async fn get_replication_reset_status(
|
||||
async fn get_replication_reset_status(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
arn: &str,
|
||||
@@ -3837,244 +3837,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a directory
|
||||
/// marker (`prefix/` with a body) in a versioned bucket is stored as the null
|
||||
/// version, like MinIO (`putOpts`: "for directory objects skip creating new
|
||||
/// versions"), and must still replicate to completion instead of staying
|
||||
/// `PENDING`.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_replicates_directory_marker_in_versioned_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-dir-marker-src";
|
||||
let target_bucket = "replication-dir-marker-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let marker_key = "dir/trailing/";
|
||||
let body = b"directory marker body";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(marker_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
put.version_id()
|
||||
.is_none_or(|id| id == "null" || id == uuid::Uuid::nil().to_string()),
|
||||
"a directory marker is the null version even in a versioned bucket: {:?}",
|
||||
put.version_id()
|
||||
);
|
||||
|
||||
wait_for_source_replication_status(&source_client, source_bucket, marker_key, "COMPLETED", false).await?;
|
||||
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body);
|
||||
let listed = target_client
|
||||
.list_object_versions()
|
||||
.bucket(target_bucket)
|
||||
.prefix(marker_key)
|
||||
.send()
|
||||
.await?;
|
||||
let marker_versions: Vec<_> = listed.versions().iter().filter(|v| v.key() == Some(marker_key)).collect();
|
||||
assert_eq!(marker_versions.len(), 1, "the marker must land exactly once: {marker_versions:?}");
|
||||
assert_eq!(
|
||||
marker_versions[0].version_id(),
|
||||
Some("null"),
|
||||
"the replica keeps the null version identity"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): permanently
|
||||
/// deleting a version whose payload lives in a data dir must leave the source
|
||||
/// clean once the purge replicates. Managed-SSE objects are never inlined and a
|
||||
/// plain object above the inline threshold takes the same layout. The version
|
||||
/// retained with a pending purge used to lose its data dir, so the purge state
|
||||
/// could never be applied (`VersionNotFound` on every retry) and the bucket
|
||||
/// stayed `BucketNotEmpty` while `ListObjectVersions` was already empty.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_version_purge_of_non_inline_object_releases_source_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("purge-datadir", true, true).await?;
|
||||
let target_arn = wait_for_remote_target_arn(&source_env, &source_bucket).await?;
|
||||
put_bucket_replication_with_delete_statuses(&source_env, &source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
let sse_key = "sse-object.bin";
|
||||
let large_key = "large-object.bin";
|
||||
let sse_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(sse_key)
|
||||
.body(ByteStream::from_static(b"encrypted source payload"))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
let large_put = source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(large_key)
|
||||
.body(ByteStream::from(vec![0x5a; 2 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let purged = [
|
||||
(sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
|
||||
(large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
|
||||
];
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
|
||||
for (key, version_id) in &purged {
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(*key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
// The purge state is applied on the source asynchronously after the target
|
||||
// acknowledges the delete; only then does the retained version go away and
|
||||
// the bucket become deletable. A listing that is empty while DeleteBucket
|
||||
// keeps answering BucketNotEmpty is exactly the regression.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let listing = source_client.list_object_versions().bucket(&source_bucket).send().await?;
|
||||
let listed = listing.versions().len() + listing.delete_markers().len();
|
||||
match source_client.delete_bucket().bucket(&source_bucket).send().await {
|
||||
Ok(_) => break,
|
||||
Err(err) if err.code() == Some("BucketNotEmpty") => {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"source bucket stayed BucketNotEmpty after the version purge replicated; \
|
||||
ListObjectVersions shows {listed} entries"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340 (not Wasabi specific): a single-part
|
||||
/// object uploaded with `x-amz-checksum-*` must reach the target with the same
|
||||
/// checksum. The outbound options keyed the stored record by algorithm name,
|
||||
/// which the target client sent as `x-amz-meta-*` user metadata, so a replica
|
||||
/// never carried a checksum although the source HEAD returned one.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_forwards_single_part_object_checksums() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-checksum-src";
|
||||
let target_bucket = "replication-checksum-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let body = b"123456789";
|
||||
let crc32_key = "checksum-crc32.txt";
|
||||
let sha256_key = "checksum-sha256.txt";
|
||||
let crc32_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(crc32_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_crc32 = crc32_put.checksum_crc32().ok_or("source PUT omitted CRC32")?.to_string();
|
||||
let sha256_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(sha256_key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
let expected_sha256 = sha256_put.checksum_sha256().ok_or("source PUT omitted SHA256")?.to_string();
|
||||
|
||||
for key in [crc32_key, sha256_key] {
|
||||
wait_for_source_replication_status(&source_client, source_bucket, key, "COMPLETED", false).await?;
|
||||
}
|
||||
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(crc32_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.checksum_crc32(), Some(expected_crc32.as_str()), "replica lost the CRC32 checksum");
|
||||
let replica = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(sha256_key)
|
||||
.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica.checksum_sha256(),
|
||||
Some(expected_sha256.as_str()),
|
||||
"replica lost the SHA256 checksum"
|
||||
);
|
||||
// The bare algorithm name must not leak as user metadata either.
|
||||
assert!(
|
||||
replica
|
||||
.metadata()
|
||||
.is_none_or(|meta| !meta.keys().any(|k| k.eq_ignore_ascii_case("sha256"))),
|
||||
"replica carries the checksum as user metadata: {:?}",
|
||||
replica.metadata()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() -> TestResult {
|
||||
init_logging();
|
||||
@@ -9289,13 +9051,11 @@ async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
fidelity["Code"], "BucketRemoteTargetVersionMismatch",
|
||||
"the failure must carry a machine-readable code: {payload}"
|
||||
);
|
||||
// The probe PUT itself succeeded (fidelity is judged from its response).
|
||||
// The mutation phases address the id the target assigned — the ledger
|
||||
// the worker records per object (rustfs/backlog#2340) — so they run and
|
||||
// pass on a drifting target, and cleanup uses the same id.
|
||||
// The probe PUT itself succeeded (fidelity is judged from its response);
|
||||
// the later mutation phases are pointless against a drifting target and
|
||||
// must be skipped, but cleanup still runs.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
// The probe PUT must carry the source version as `?versionId=` — the
|
||||
|
||||
@@ -31,22 +31,17 @@
|
||||
//! Adding a target behavior the fleet has shown: add the mode to the fake
|
||||
//! target, add a row here, and record any cell that is red before the fix.
|
||||
|
||||
use crate::common::{init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::{OdmEnvOptions, OdmTestEnv, fake_source_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, replication_fast_env};
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::fake_source_client;
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, delete_bucket_replication, enable_bucket_versioning,
|
||||
get_replication_reset_status, put_bucket_replication, put_bucket_replication_with_delete_statuses,
|
||||
set_replication_target_with_options, start_bucket_replication_reset,
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, put_bucket_replication,
|
||||
set_replication_target_with_options,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use aws_sdk_s3::types::{
|
||||
Checksum, ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ObjectAttributes, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, Tag, Tagging,
|
||||
};
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockMode};
|
||||
use bytes::Bytes;
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -68,10 +63,7 @@ enum TargetMode {
|
||||
/// Object Lock parameters must carry `Content-MD5` or `x-amz-checksum-*`.
|
||||
RequireChecksumWithObjectLock,
|
||||
/// AWS S3 / Wasabi / Impossible Cloud: mints its own version ids
|
||||
/// (rustfs/backlog#2085) and, like Wasabi, answers NoSuchVersion to a
|
||||
/// DELETE of an id it never had (rustfs/backlog#2340). Data must still
|
||||
/// land, and every version-addressed mutation must resolve the replica
|
||||
/// through the target-version ledger.
|
||||
/// (rustfs/backlog#2085). Data must still land.
|
||||
MintOwnVersionIds,
|
||||
}
|
||||
|
||||
@@ -88,10 +80,7 @@ impl TargetMode {
|
||||
TargetMode::Baseline => {}
|
||||
TargetMode::RejectAwsChunked => target.reject_aws_chunked_uploads(true),
|
||||
TargetMode::RequireChecksumWithObjectLock => target.require_checksum_for_object_lock(true),
|
||||
TargetMode::MintOwnVersionIds => {
|
||||
target.assign_own_version_ids(true);
|
||||
target.reject_unknown_version_deletes(true);
|
||||
}
|
||||
TargetMode::MintOwnVersionIds => target.assign_own_version_ids(true),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,23 +110,16 @@ enum ObjectShape {
|
||||
/// Two-part multipart upload with a GOVERNANCE retention period; the
|
||||
/// lock headers travel on CreateMultipartUpload, which has no body.
|
||||
LockedMultipart,
|
||||
/// ODM stores two local parts while preserving a single-PUT source's MD5 ETag.
|
||||
OdmPreservedMd5Multipart,
|
||||
/// Single-part object uploaded with `x-amz-checksum-sha256`; the replica
|
||||
/// must carry the same header (rustfs/backlog#2340).
|
||||
Checksummed,
|
||||
}
|
||||
|
||||
impl ObjectShape {
|
||||
const ALL: [ObjectShape; 8] = [
|
||||
const ALL: [ObjectShape; 6] = [
|
||||
ObjectShape::Empty,
|
||||
ObjectShape::Plain,
|
||||
ObjectShape::Retention,
|
||||
ObjectShape::LegalHold,
|
||||
ObjectShape::Multipart,
|
||||
ObjectShape::LockedMultipart,
|
||||
ObjectShape::OdmPreservedMd5Multipart,
|
||||
ObjectShape::Checksummed,
|
||||
];
|
||||
|
||||
fn key(self) -> &'static str {
|
||||
@@ -148,17 +130,6 @@ impl ObjectShape {
|
||||
ObjectShape::LegalHold => "matrix/legal-hold.bin",
|
||||
ObjectShape::Multipart => "matrix/multipart.bin",
|
||||
ObjectShape::LockedMultipart => "matrix/locked-multipart.bin",
|
||||
ObjectShape::OdmPreservedMd5Multipart => "matrix/odm-preserved-md5.bin",
|
||||
ObjectShape::Checksummed => "matrix/checksummed.bin",
|
||||
}
|
||||
}
|
||||
|
||||
/// The `x-amz-checksum-*` header the source stored and every upload of
|
||||
/// the replica must repeat.
|
||||
fn forwarded_checksum_header(self) -> Option<&'static str> {
|
||||
match self {
|
||||
ObjectShape::Checksummed => Some("x-amz-checksum-sha256"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,8 +139,7 @@ impl ObjectShape {
|
||||
|
||||
/// Upload the shape to the source and return the bytes the target must
|
||||
/// end up holding.
|
||||
async fn put(self, env: &OdmTestEnv, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
let client = &env.client;
|
||||
async fn put(self, client: &Client, bucket: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
let key = self.key();
|
||||
match self {
|
||||
ObjectShape::Empty => {
|
||||
@@ -220,19 +190,6 @@ impl ObjectShape {
|
||||
}
|
||||
ObjectShape::Multipart => multipart_put(client, bucket, key, 0x44, false).await,
|
||||
ObjectShape::LockedMultipart => multipart_put(client, bucket, key, 0x55, true).await,
|
||||
ObjectShape::OdmPreservedMd5Multipart => odm_preserved_md5_multipart(env, bucket, key).await,
|
||||
ObjectShape::Checksummed => {
|
||||
let body = payload(40 * 1024, 0x66);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -262,595 +219,6 @@ fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation {
|
||||
.unwrap_or(Expectation::Completed)
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340: a target that mints its own version ids (Wasabi,
|
||||
/// AWS S3) answers 404 to a HEAD by the source uuid, which the worker used to
|
||||
/// read as "replica missing" and re-drive the PUT — one more target version
|
||||
/// per heal, MRF retry or resync. Two re-drive shapes, both must converge on
|
||||
/// the single version the first PUT created:
|
||||
/// - the first PUT lands but its response is lost, so the object is FAILED
|
||||
/// and the scanner heal pass re-drives it;
|
||||
/// - an existing-object resync re-drives a COMPLETED object unconditionally.
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-redrive-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what re-drives a FAILED object.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-mint-own-redrive-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
// Teach the worker the target's identity contract with one ordinary
|
||||
// write, exactly as production learns it (the PUT response carries the
|
||||
// minted id).
|
||||
let probe_key = "redrive/identity-probe.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(probe_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, probe_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
|
||||
// Shape 1: the PUT is stored, its response never arrives, heal re-drives.
|
||||
let heal_key = "redrive/heal.bin";
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, heal_key, FakeTargetFault::DisconnectAfterResponse, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(heal_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x02)))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replication_status_and_single_version(&source_client, source_bucket, &target, &target_bucket, heal_key).await?;
|
||||
|
||||
// Shape 2: an existing-object resync re-drives a COMPLETED object.
|
||||
let resync_key = "redrive/resync.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(resync_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x03)))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, resync_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let (reset_arn, _reset_id) = start_bucket_replication_reset(source_env, source_bucket).await?;
|
||||
assert_eq!(reset_arn, target_arn);
|
||||
let resync = async {
|
||||
loop {
|
||||
let status = get_replication_reset_status(source_env, source_bucket, &target_arn).await?;
|
||||
if let Some(entry) = status.targets.iter().find(|entry| entry.arn == target_arn)
|
||||
&& entry.status == "Completed"
|
||||
{
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(entry.replicated_count);
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
let replicated = timeout(Duration::from_secs(90), resync)
|
||||
.await
|
||||
.map_err(|_| "existing-object resync did not complete within 90 seconds")??;
|
||||
assert!(replicated >= 3, "resync must count the located replicas as replicated, got {replicated}");
|
||||
for key in [probe_key, heal_key, resync_key] {
|
||||
let versions = target.stored_versions(&target_bucket, key);
|
||||
assert_eq!(
|
||||
versions.len(),
|
||||
1,
|
||||
"{key}: a re-drive against a target that mints its own version ids must not mint another one: {versions:?}"
|
||||
);
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340 (target-version ledger): on a target that mints its own
|
||||
/// version ids and answers NoSuchVersion to an unknown id (the Wasabi shape),
|
||||
/// every version-addressed mutation must land on the version the target
|
||||
/// assigned, which the replication PUT recorded on the source:
|
||||
/// - a tag update changes the existing target version, no new version;
|
||||
/// - a retention extension and legal hold ON/OFF change that version too;
|
||||
/// - a permanent delete of the older of two same-content generations removes
|
||||
/// exactly that replica and keeps the live one (content identity alone
|
||||
/// could not tell them apart).
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_addresses_mutations_through_the_ledger() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-ledger-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
TargetMode::MintOwnVersionIds.apply(&target);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass retries a purge the first attempt lost.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-mint-own-ledger-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
let target_client = fake_source_client(&target);
|
||||
|
||||
// Tag update on an existing version.
|
||||
let tag_key = "ledger/tags.bin";
|
||||
let tagged = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(tag_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
let tag_source_version = tagged.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, tag_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let tag_target_version = single_target_version(&target, &target_bucket, tag_key)?;
|
||||
source_client
|
||||
.put_object_tagging()
|
||||
.bucket(source_bucket)
|
||||
.key(tag_key)
|
||||
.version_id(&tag_source_version)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("phase").value("after").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("tag update on the existing target version", || async {
|
||||
let tags = target_client
|
||||
.get_object_tagging()
|
||||
.bucket(&target_bucket)
|
||||
.key(tag_key)
|
||||
.version_id(&tag_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(tags
|
||||
.tag_set()
|
||||
.iter()
|
||||
.any(|tag| tag.key() == "phase" && tag.value() == "after"))
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, tag_key, &tag_target_version).await?;
|
||||
|
||||
// Retention extension and legal hold on an existing version.
|
||||
let lock_key = "ledger/lock.bin";
|
||||
let locked = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x02)))
|
||||
.object_lock_mode(ObjectLockMode::Governance)
|
||||
.object_lock_retain_until_date(retain_until())
|
||||
.send()
|
||||
.await?;
|
||||
let lock_source_version = locked.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, lock_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let lock_target_version = single_target_version(&target, &target_bucket, lock_key)?;
|
||||
let extended = DateTime::from_secs(retain_until().secs() + 86_400);
|
||||
source_client
|
||||
.put_object_retention()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.retention(
|
||||
ObjectLockRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.retain_until_date(extended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
source_client
|
||||
.put_object_legal_hold()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::On).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("retention extension and legal hold on the existing target version", || async {
|
||||
let head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(head.object_lock_retain_until_date().map(|date| date.secs()) == Some(extended.secs())
|
||||
&& head.object_lock_legal_hold_status() == Some(&ObjectLockLegalHoldStatus::On))
|
||||
})
|
||||
.await?;
|
||||
source_client
|
||||
.put_object_legal_hold()
|
||||
.bucket(source_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_source_version)
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(ObjectLockLegalHoldStatus::Off).build())
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("legal hold removal on the existing target version", || async {
|
||||
let head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(lock_key)
|
||||
.version_id(&lock_target_version)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(head.object_lock_legal_hold_status() == Some(&ObjectLockLegalHoldStatus::Off))
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, lock_key, &lock_target_version).await?;
|
||||
|
||||
// Permanent delete of the older of two same-content generations.
|
||||
let generations_key = "ledger/generations.bin";
|
||||
let body = payload(4 * 1024, 0x03);
|
||||
let older = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let older_version = older.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, generations_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let older_replica = single_target_version(&target, &target_bucket, generations_key)?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.body(ByteStream::from(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, generations_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
wait_until("both generations replicated", || async {
|
||||
Ok(target.stored_versions(&target_bucket, generations_key).len() == 2)
|
||||
})
|
||||
.await?;
|
||||
let newer_replica = target
|
||||
.stored_versions(&target_bucket, generations_key)
|
||||
.into_iter()
|
||||
.map(|(version_id, _)| version_id)
|
||||
.find(|version_id| version_id != &older_replica)
|
||||
.ok_or("the second generation must have its own target version")?;
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(generations_key)
|
||||
.version_id(&older_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("permanent delete of the older generation's replica", || async {
|
||||
let versions: Vec<String> = target
|
||||
.stored_versions(&target_bucket, generations_key)
|
||||
.into_iter()
|
||||
.map(|(version_id, _)| version_id)
|
||||
.collect();
|
||||
Ok(versions == [newer_replica.clone()])
|
||||
})
|
||||
.await?;
|
||||
assert_stable_single_version(&target, &target_bucket, generations_key, &newer_replica).await?;
|
||||
|
||||
// No mutation above may have gone out as a re-PUT: one upload per key.
|
||||
for key in [tag_key, lock_key] {
|
||||
let puts = target
|
||||
.requests()
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(key) && record.operation == FakeTargetOperation::PutObject)
|
||||
.count();
|
||||
assert_eq!(
|
||||
puts, 1,
|
||||
"{key}: a metadata update must not re-PUT the object on a target that mints its own ids"
|
||||
);
|
||||
}
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340 (pending purge lifecycle): a permanent delete whose
|
||||
/// replication keeps failing leaves the version in xl.meta as a PENDING purge,
|
||||
/// hidden from listings. Once the bucket's replication configuration is
|
||||
/// removed nothing can ever confirm that purge remotely, so the delete worker
|
||||
/// must settle it locally (abandoned, with the replica left on the former
|
||||
/// target) — otherwise the bucket stays `BucketNotEmpty` forever with a
|
||||
/// residue the client cannot see.
|
||||
#[tokio::test]
|
||||
async fn matrix_removed_replication_config_abandons_pending_purge() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-abandoned-purge-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
TargetMode::MintOwnVersionIds.apply(&target);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what revisits a pending purge.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-abandoned-purge-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
|
||||
let key = "purge/orphaned.bin";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x07)))
|
||||
.send()
|
||||
.await?;
|
||||
let source_version = put.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let replica = single_target_version(&target, &target_bucket, key)?;
|
||||
|
||||
// The target refuses every purge: the version stays a pending purge.
|
||||
// More refusals than any scanner cycle can consume within the test.
|
||||
target.inject_for_key(FakeTargetOperation::DeleteObject, key, FakeTargetFault::ResponseStatus(503), 4_000);
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.version_id(&source_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("the refused purge to reach the target at least once", || async {
|
||||
Ok(target.count_requests(FakeTargetOperation::DeleteObject, key) >= 1)
|
||||
})
|
||||
.await?;
|
||||
let listed = source_client.list_object_versions().bucket(source_bucket).send().await?;
|
||||
assert!(
|
||||
listed.versions().is_empty() && listed.delete_markers().is_empty(),
|
||||
"a pending purge is hidden from listings: {listed:?}"
|
||||
);
|
||||
let blocked = source_client.delete_bucket().bucket(source_bucket).send().await;
|
||||
assert!(
|
||||
blocked
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| err.as_service_error())
|
||||
.is_some_and(|err| err.code() == Some("BucketNotEmpty")),
|
||||
"the hidden pending purge must block DeleteBucket while the target is still configured: {blocked:?}"
|
||||
);
|
||||
|
||||
// Removing the replication configuration orphans the purge; the scanner
|
||||
// heal pass must settle it locally so the bucket becomes deletable.
|
||||
let response = delete_bucket_replication(source_env, source_bucket).await?;
|
||||
assert!(response.status().is_success(), "DeleteBucketReplication: {}", response.status());
|
||||
wait_until("DeleteBucket to succeed once the orphaned purge is abandoned", || async {
|
||||
match source_client.delete_bucket().bucket(source_bucket).send().await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if err.as_service_error().is_some_and(|err| err.code() == Some("BucketNotEmpty")) => Ok(false),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// Abandoned means abandoned: the replica stays on the former target and,
|
||||
// once the attempts in flight at removal time have drained, no further
|
||||
// purge attempts are sent to it.
|
||||
assert_eq!(
|
||||
single_target_version(&target, &target_bucket, key)?,
|
||||
replica,
|
||||
"an abandoned purge must not touch the replica on the former target"
|
||||
);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
let settled = target.count_requests(FakeTargetOperation::DeleteObject, key);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
assert_eq!(
|
||||
target.count_requests(FakeTargetOperation::DeleteObject, key),
|
||||
settled,
|
||||
"purge attempts must stop once the target is no longer configured"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn single_target_version(target: &FakeS3Target, target_bucket: &str, key: &str) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
match versions.as_slice() {
|
||||
[(version_id, false)] => Ok(version_id.clone()),
|
||||
other => Err(format!("{key}: expected exactly one live target version, got {other:?}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The target keeps holding exactly `version_id` for a few scanner cycles: a
|
||||
/// re-driven PUT or a wrong delete would show up here.
|
||||
async fn assert_stable_single_version(target: &FakeS3Target, target_bucket: &str, key: &str, version_id: &str) -> TestResult {
|
||||
for _ in 0..8 {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
if versions.len() != 1 || versions[0].0 != version_id {
|
||||
return Err(
|
||||
format!("{key}: target versions drifted from the single expected replica {version_id}: {versions:?}").into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(what: &str, mut probe: F) -> TestResult
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<bool, Box<dyn Error + Send + Sync>>>,
|
||||
{
|
||||
let wait = async {
|
||||
loop {
|
||||
if probe().await? {
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
timeout(Duration::from_secs(90), wait)
|
||||
.await
|
||||
.map_err(|_| format!("{what} did not happen within 90 seconds"))?
|
||||
}
|
||||
|
||||
/// Wait until `key` is COMPLETED on the source and, for the observation
|
||||
/// window after that, the target still holds exactly one live version of it.
|
||||
async fn wait_for_replication_status_and_single_version(
|
||||
source_client: &Client,
|
||||
source_bucket: &str,
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
key: &str,
|
||||
) -> TestResult {
|
||||
// The lost PUT response first settles the object FAILED; only the next
|
||||
// scanner heal pass can turn that into COMPLETED, so FAILED is transient
|
||||
// here and the wait is for COMPLETED alone.
|
||||
let converged = async {
|
||||
loop {
|
||||
let head = source_client.head_object().bucket(source_bucket).key(key).send().await?;
|
||||
if head.replication_status().is_some_and(|status| status.as_str() == "COMPLETED") {
|
||||
return Ok::<_, Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
};
|
||||
timeout(Duration::from_secs(90), converged)
|
||||
.await
|
||||
.map_err(|_| format!("{key}: heal re-drive did not converge to COMPLETED within 90 seconds"))??;
|
||||
// The heal pass keeps visiting the key for a few scanner cycles; a
|
||||
// duplicate would show up here as a second stored version.
|
||||
for _ in 0..12 {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
assert_eq!(versions.len(), 1, "{key}: target minted another version on re-drive: {versions:?}");
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn matrix_baseline_target() -> TestResult {
|
||||
run_row(TargetMode::Baseline).await
|
||||
@@ -902,15 +270,11 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
mode.apply(&target);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = format!("matrix-{}-src", mode.slug());
|
||||
let source_client = source_env.create_s3_client();
|
||||
@@ -920,9 +284,9 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, &source_bucket).await?;
|
||||
enable_bucket_versioning(&source_env, &source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
&source_env,
|
||||
&source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
@@ -935,21 +299,14 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, &source_bucket, &target_arn).await?;
|
||||
put_bucket_replication(&source_env, &source_bucket, &target_arn).await?;
|
||||
|
||||
let target_client = fake_source_client(&target);
|
||||
let mut failures = Vec::new();
|
||||
for shape in ObjectShape::ALL {
|
||||
let cell = format!("{}/{:?}", mode.slug(), shape);
|
||||
let expected_body = shape.put(&env, &source_bucket).await?;
|
||||
let expected_body = shape.put(&source_client, &source_bucket).await?;
|
||||
let status = wait_for_terminal_replication_status(&source_client, &source_bucket, shape.key()).await?;
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
assert_eq!(
|
||||
env.source.count_requests(FakeTargetOperation::GetObject, shape.key()),
|
||||
2,
|
||||
"one passthrough GET plus one background pull; replication must read the persisted local parts"
|
||||
);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let outcome = match expectation(mode, shape) {
|
||||
Expectation::Completed => {
|
||||
@@ -1022,36 +379,6 @@ async fn check_completed_cell(
|
||||
if uploads.is_empty() {
|
||||
return Err("no upload reached the target although the source reports COMPLETED".into());
|
||||
}
|
||||
if shape == ObjectShape::OdmPreservedMd5Multipart {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(shape.key()))
|
||||
.collect();
|
||||
for operation in [
|
||||
FakeTargetOperation::CreateMultipartUpload,
|
||||
FakeTargetOperation::CompleteMultipartUpload,
|
||||
] {
|
||||
if !key_requests.iter().any(|record| record.operation == operation) {
|
||||
return Err(format!("preserved-MD5 multipart object did not use {operation:?}").into());
|
||||
}
|
||||
}
|
||||
if key_requests
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
{
|
||||
return Err("preserved-MD5 multipart object used a single PutObject".into());
|
||||
}
|
||||
let mut part_numbers: Vec<_> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.map(|record| record.part_number)
|
||||
.collect();
|
||||
part_numbers.sort_unstable();
|
||||
part_numbers.dedup();
|
||||
if part_numbers != [Some(1), Some(2)] {
|
||||
return Err(format!("preserved-MD5 multipart object uploaded unexpected parts: {part_numbers:?}").into());
|
||||
}
|
||||
}
|
||||
if let Some(framed) = uploads.iter().find(|record| record.transport.aws_chunked) {
|
||||
return Err(format!("{cell}: an upload went out aws-chunked (rustfs#6853 framing): {framed:?}").into());
|
||||
}
|
||||
@@ -1074,19 +401,6 @@ async fn check_completed_cell(
|
||||
}) {
|
||||
return Err(format!("a locked PutObject went out without any integrity header (rustfs#7082): {bare:?}").into());
|
||||
}
|
||||
// rustfs/backlog#2340 contract: a source checksum reaches the target as
|
||||
// the `x-amz-checksum-*` header, not as user metadata; every PutObject of
|
||||
// the shape carries it.
|
||||
if let Some(header) = shape.forwarded_checksum_header()
|
||||
&& let Some(missing) = uploads.iter().find(|record| {
|
||||
record.operation == FakeTargetOperation::PutObject
|
||||
&& !record.transport.checksum_headers.iter().any(|name| name == header)
|
||||
})
|
||||
{
|
||||
return Err(
|
||||
format!("a PutObject went out without the source's {header} header (rustfs/backlog#2340): {missing:?}").into(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1141,71 +455,6 @@ async fn wait_for_terminal_replication_status(
|
||||
}
|
||||
}
|
||||
|
||||
async fn odm_preserved_md5_multipart(env: &OdmTestEnv, bucket: &str, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let origin_bucket = format!("{bucket}-origin");
|
||||
env.source.create_bucket_with_mode(&origin_bucket, BucketMode::Unversioned);
|
||||
let mut spec = env.fake_source_spec(&origin_bucket);
|
||||
// Below the 16 MiB inline default the pull is one tee'd PUT with a single
|
||||
// part; force the passthrough + background multipart write-back instead.
|
||||
spec.policy.inline_max_bytes = 4096;
|
||||
spec.policy.multipart_part_size_bytes = PART_SIZE as u64;
|
||||
spec.policy.preserve_etag = true;
|
||||
env.configure_and_wait(bucket, &spec).await?;
|
||||
|
||||
// A normal source PUT produces the MD5 ETag; only ODM chooses the local parts.
|
||||
let body = payload(PART_SIZE + 4096, 0x66);
|
||||
let source_put = env
|
||||
.source_client()
|
||||
.put_object()
|
||||
.bucket(&origin_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let source_etag = source_put.e_tag().ok_or("source PUT omitted its ETag")?.trim_matches('"');
|
||||
assert_eq!(source_etag.len(), 32, "source fixture must have a single-PUT MD5 ETag");
|
||||
assert!(source_etag.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
|
||||
let pulled = env.raw_get(bucket, key).await?;
|
||||
assert_eq!(pulled.status, 200, "{}", String::from_utf8_lossy(&pulled.body));
|
||||
assert_eq!(pulled.body, body);
|
||||
assert!(
|
||||
env.wait_local_listed(bucket, key, Duration::from_secs(30)).await?,
|
||||
"ODM must persist the object"
|
||||
);
|
||||
let attributes = env
|
||||
.client
|
||||
.get_object_attributes()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::Checksum)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(attributes.e_tag().map(|etag| etag.trim_matches('"')), Some(source_etag));
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or("the ODM copy must expose its two local parts")?;
|
||||
assert_eq!(parts.total_parts_count(), Some(2));
|
||||
assert_eq!(
|
||||
parts
|
||||
.parts()
|
||||
.iter()
|
||||
.map(|part| (part.part_number(), part.size()))
|
||||
.collect::<Vec<_>>(),
|
||||
[(Some(1), Some(PART_SIZE as i64)), (Some(2), Some(4096))]
|
||||
);
|
||||
assert!(
|
||||
attributes
|
||||
.checksum()
|
||||
.is_none_or(|checksum| checksum == &Checksum::builder().build()),
|
||||
"multipart routing must work without an object checksum record"
|
||||
);
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
async fn multipart_put(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
|
||||
signed_request,
|
||||
};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject, fake_source_client};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
|
||||
};
|
||||
@@ -26,10 +25,9 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectAttributes, ObjectLockConfiguration,
|
||||
ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging,
|
||||
VersioningConfiguration,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -1207,724 +1205,3 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// rc.5 multipart layouts under the current build (backlog#2147 follow-up to
|
||||
// rustfs#7305)
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// rustfs#7305 changed `ObjectInfo::is_multipart` to consult the stored part
|
||||
// list before the ETag shape. Every earlier check of that change used
|
||||
// synthetic metadata; this scenario writes the layouts with the published
|
||||
// rc.5 binary and then reads, describes, and replicates them with the
|
||||
// current build on the same data directory.
|
||||
|
||||
const LAYOUT_PLAIN_BUCKET: &str = "upgrade-layout-plain";
|
||||
const LAYOUT_ENCRYPTED_BUCKET: &str = "upgrade-layout-encrypted";
|
||||
const LAYOUT_REPLICA_BUCKET: &str = "upgrade-layout-replica";
|
||||
const LAYOUT_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LAYOUT_TAIL_SIZE: usize = 1024 * 1024 + 4096;
|
||||
const LAYOUT_SSEC_KEY: &str = "0123456789abcdef0123456789abcdef";
|
||||
const LAYOUT_REPLICATION_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
|
||||
struct LayoutCase {
|
||||
bucket: &'static str,
|
||||
key: &'static str,
|
||||
/// Empty for a single PUT.
|
||||
part_sizes: Vec<usize>,
|
||||
body: Vec<u8>,
|
||||
ssec: bool,
|
||||
/// `false` for layouts whose replication is a known pre-existing failure;
|
||||
/// their outcome is logged, not asserted.
|
||||
assert_replication: bool,
|
||||
/// Recorded from the rc.5 writer.
|
||||
rc5_etag: String,
|
||||
/// Whether rc.5 reported `ObjectParts` for the object.
|
||||
rc5_reported_parts: Option<usize>,
|
||||
}
|
||||
|
||||
impl LayoutCase {
|
||||
fn is_multipart_layout(&self) -> bool {
|
||||
self.part_sizes.len() > 1
|
||||
}
|
||||
|
||||
fn label(&self) -> String {
|
||||
format!("{}/{}", self.bucket, self.key)
|
||||
}
|
||||
}
|
||||
|
||||
fn layout_noise(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut state = seed ^ 0x9E37_79B9_7F4A_7C15;
|
||||
(0..len)
|
||||
.map(|_| {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
(state >> 24) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn layout_text(len: usize, seed: u64) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len + 64);
|
||||
let mut line = 0u64;
|
||||
while out.len() < len {
|
||||
out.extend_from_slice(format!("rc5 legacy layout seed={seed} line={line} lorem ipsum dolor sit amet\n").as_bytes());
|
||||
line += 1;
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
}
|
||||
|
||||
fn layout_ssec_key_md5() -> String {
|
||||
use md5::{Digest as _, Md5};
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(LAYOUT_SSEC_KEY.as_bytes());
|
||||
base64_simd::STANDARD.encode_to_string(hasher.finalize())
|
||||
}
|
||||
|
||||
fn layout_ssec_key() -> String {
|
||||
base64_simd::STANDARD.encode_to_string(LAYOUT_SSEC_KEY)
|
||||
}
|
||||
|
||||
async fn layout_head(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::head_object::HeadObjectOutput, BoxError> {
|
||||
let request = client.head_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_get(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
) -> Result<aws_sdk_s3::operation::get_object::GetObjectOutput, BoxError> {
|
||||
let request = client.get_object().bucket(case.bucket).key(case.key);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = request.set_range(range).set_part_number(part_number);
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn layout_attributes(
|
||||
client: &Client,
|
||||
case: &LayoutCase,
|
||||
) -> Result<aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput, BoxError> {
|
||||
let request = client
|
||||
.get_object_attributes()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.object_attributes(ObjectAttributes::Etag)
|
||||
.object_attributes(ObjectAttributes::ObjectParts)
|
||||
.object_attributes(ObjectAttributes::ObjectSize)
|
||||
.max_parts(100);
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
/// Write `case` with the rc.5 client; single PUT when `part_sizes` is empty.
|
||||
async fn layout_write(client: &Client, case: &LayoutCase) -> Result<(), BoxError> {
|
||||
let content_type = "text/plain";
|
||||
if case.part_sizes.is_empty() {
|
||||
let request = client
|
||||
.put_object()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type)
|
||||
.body(ByteStream::from(case.body.clone()));
|
||||
let request = if case.ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
request
|
||||
};
|
||||
request.send().await?;
|
||||
return Ok(());
|
||||
}
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.content_type(content_type);
|
||||
let create = if case.ssec {
|
||||
create
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
create
|
||||
};
|
||||
let created = create.send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
let mut completed = Vec::with_capacity(case.part_sizes.len());
|
||||
let mut offset = 0usize;
|
||||
for (index, size) in case.part_sizes.iter().enumerate() {
|
||||
let part_number = i32::try_from(index + 1)?;
|
||||
let chunk = case.body[offset..offset + size].to_vec();
|
||||
offset += size;
|
||||
let upload = client
|
||||
.upload_part()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(chunk));
|
||||
let upload = if case.ssec {
|
||||
upload
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(layout_ssec_key())
|
||||
.sse_customer_key_md5(layout_ssec_key_md5())
|
||||
} else {
|
||||
upload
|
||||
};
|
||||
let uploaded = upload.send().await?;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().ok_or("UploadPart omitted ETag")?)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key(case.key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn layout_cases() -> Vec<LayoutCase> {
|
||||
let two = vec![LAYOUT_PART_SIZE, LAYOUT_TAIL_SIZE];
|
||||
let three = vec![LAYOUT_PART_SIZE, LAYOUT_PART_SIZE, 4096];
|
||||
let total = |sizes: &[usize]| sizes.iter().sum::<usize>();
|
||||
let case = |bucket, key, part_sizes: Vec<usize>, body: Vec<u8>, ssec| LayoutCase {
|
||||
bucket,
|
||||
key,
|
||||
part_sizes,
|
||||
body,
|
||||
ssec,
|
||||
assert_replication: true,
|
||||
rc5_etag: String::new(),
|
||||
rc5_reported_parts: None,
|
||||
};
|
||||
vec![
|
||||
case(LAYOUT_PLAIN_BUCKET, "plain/single.bin", vec![], layout_noise(1024 * 1024 + 17, 1), false),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 2),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 3),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-single.txt",
|
||||
vec![],
|
||||
layout_text(1024 * 1024 + 17, 4),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 5),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 6),
|
||||
true,
|
||||
),
|
||||
// SSE-C passthrough replicates the stored ciphertext part by part; a
|
||||
// compressible first part is stored well below 5 MiB and a standard
|
||||
// target rejects it with EntityTooSmall. rc.5 fails the same way (see
|
||||
// `rc5_baseline_replicates_multipart_layouts`), so the outcome is
|
||||
// recorded rather than asserted here; tracked as rustfs/backlog#2363.
|
||||
LayoutCase {
|
||||
assert_replication: false,
|
||||
..case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 7),
|
||||
true,
|
||||
)
|
||||
},
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/single.bin",
|
||||
vec![],
|
||||
layout_noise(1024 * 1024 + 17, 8),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-2.bin",
|
||||
two.clone(),
|
||||
layout_noise(total(&two), 9),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/multipart-3.bin",
|
||||
three.clone(),
|
||||
layout_noise(total(&three), 10),
|
||||
false,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 11),
|
||||
false,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn layout_server_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = bucket_config_server_env();
|
||||
env.push(("RUSTFS_COMPRESSION_ENABLED", "true"));
|
||||
env.push(("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"));
|
||||
env
|
||||
}
|
||||
|
||||
fn layout_reported_parts(attributes: &aws_sdk_s3::operation::get_object_attributes::GetObjectAttributesOutput) -> Option<usize> {
|
||||
attributes.object_parts().map(|parts| parts.parts().len())
|
||||
}
|
||||
|
||||
async fn assert_layout_readable(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let head = layout_head(client, case).await?;
|
||||
assert_eq!(
|
||||
head.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: the ETag written by rc.5 must be reported unchanged"
|
||||
);
|
||||
assert_eq!(
|
||||
head.content_length(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: HEAD content length"
|
||||
);
|
||||
|
||||
let full = layout_get(client, case, None, None).await?.body.collect().await?.into_bytes();
|
||||
assert_eq!(full.len(), case.body.len(), "{context}: {label}: full GET length");
|
||||
assert!(full == case.body, "{context}: {label}: full GET body must equal the rc.5 upload");
|
||||
|
||||
if case.is_multipart_layout() {
|
||||
let first = case.part_sizes[0];
|
||||
let range = format!("bytes={}-{}", first - 32, first + 31);
|
||||
let crossing = layout_get(client, case, Some(range), None)
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert!(
|
||||
crossing == case.body[first - 32..first + 32],
|
||||
"{context}: {label}: range across the first part boundary"
|
||||
);
|
||||
let tail_start: usize = case.part_sizes[..case.part_sizes.len() - 1].iter().sum();
|
||||
let last_number = i32::try_from(case.part_sizes.len())?;
|
||||
let last = layout_get(client, case, None, Some(last_number)).await?;
|
||||
assert_eq!(
|
||||
last.content_length(),
|
||||
Some(i64::try_from(case.part_sizes[case.part_sizes.len() - 1])?),
|
||||
"{context}: {label}: partNumber={last_number} length"
|
||||
);
|
||||
let last_body = last.body.collect().await?.into_bytes();
|
||||
assert!(
|
||||
last_body == case.body[tail_start..],
|
||||
"{context}: {label}: partNumber={last_number} body must be the stored last part"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_layout_attributes(client: &Client, case: &LayoutCase, context: &str) -> TestResult {
|
||||
let label = case.label();
|
||||
let attributes = layout_attributes(client, case).await?;
|
||||
assert_eq!(
|
||||
attributes.e_tag().map(|etag| etag.trim_matches('"')),
|
||||
Some(case.rc5_etag.as_str()),
|
||||
"{context}: {label}: attributes ETag"
|
||||
);
|
||||
assert_eq!(
|
||||
attributes.object_size(),
|
||||
Some(i64::try_from(case.body.len())?),
|
||||
"{context}: {label}: attributes ObjectSize"
|
||||
);
|
||||
if case.is_multipart_layout() {
|
||||
let parts = attributes
|
||||
.object_parts()
|
||||
.ok_or_else(|| format!("{context}: {label}: multipart layout must expose ObjectParts"))?;
|
||||
assert_eq!(
|
||||
parts.total_parts_count(),
|
||||
Some(i32::try_from(case.part_sizes.len())?),
|
||||
"{context}: {label}: TotalPartsCount"
|
||||
);
|
||||
let observed: Vec<(Option<i32>, Option<i64>)> =
|
||||
parts.parts().iter().map(|part| (part.part_number(), part.size())).collect();
|
||||
let expected: Vec<(Option<i32>, Option<i64>)> = case
|
||||
.part_sizes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, size)| (Some(index as i32 + 1), Some(*size as i64)))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
observed, expected,
|
||||
"{context}: {label}: ObjectParts must report the plaintext part layout"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
attributes.object_parts().is_none_or(|parts| parts.parts().is_empty()),
|
||||
"{context}: {label}: a single PUT must not report stored parts"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_layout_replication_rule(env: &RustFSTestEnvironment, bucket: &str, arn: &str) -> TestResult {
|
||||
let body = format!(
|
||||
r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Role></Role>
|
||||
<Rule>
|
||||
<ID>legacy-layouts</ID>
|
||||
<Priority>1</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
</ReplicationConfiguration>"#
|
||||
);
|
||||
let url = format!("{}/{bucket}?replication", env.url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(body.into_bytes()),
|
||||
Some("application/xml"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("put replication rule on {bucket} failed: {status} {body}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for the existing-object replication of `case` to reach a terminal
|
||||
/// status and return it (`COMPLETED` or `FAILED`).
|
||||
async fn wait_layout_replication_terminal(client: &Client, case: &LayoutCase) -> Result<String, BoxError> {
|
||||
let deadline = Instant::now() + LAYOUT_REPLICATION_TIMEOUT;
|
||||
loop {
|
||||
let head = layout_head(client, case).await?;
|
||||
let status = head.replication_status().map(|status| status.as_str().to_string());
|
||||
if matches!(status.as_deref(), Some("COMPLETED") | Some("FAILED")) {
|
||||
return Ok(status.unwrap_or_default());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"{}: existing-object replication never reached a terminal status; last {status:?}",
|
||||
case.label()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct LayoutTransport {
|
||||
status: String,
|
||||
uploaded_parts: Vec<i32>,
|
||||
single_puts: usize,
|
||||
completes: usize,
|
||||
/// Raw per-key journal in target order: (sequence, operation, part number,
|
||||
/// upload id), so duplicate drives can be told apart from retries.
|
||||
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
|
||||
}
|
||||
|
||||
/// Configure every layout bucket to replicate its existing objects to a fresh
|
||||
/// fake target, wait for each case to settle, and report the transport the
|
||||
/// target observed per case.
|
||||
async fn replicate_layouts(
|
||||
env: &RustFSTestEnvironment,
|
||||
client: &Client,
|
||||
cases: &[LayoutCase],
|
||||
) -> Result<(FakeS3Target, Vec<LayoutTransport>), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(LAYOUT_REPLICA_BUCKET);
|
||||
for bucket in [LAYOUT_PLAIN_BUCKET, LAYOUT_ENCRYPTED_BUCKET] {
|
||||
let arn = set_replication_target_with_options(
|
||||
env,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: LAYOUT_REPLICA_BUCKET,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_layout_replication_rule(env, bucket, &arn).await?;
|
||||
}
|
||||
let mut statuses = Vec::with_capacity(cases.len());
|
||||
for case in cases {
|
||||
statuses.push(wait_layout_replication_terminal(client, case).await?);
|
||||
}
|
||||
let journal = target.requests();
|
||||
let mut transports = Vec::with_capacity(cases.len());
|
||||
for (case, status) in cases.iter().zip(statuses) {
|
||||
let key_requests: Vec<_> = journal
|
||||
.iter()
|
||||
.filter(|record| record.key.as_deref() == Some(case.key))
|
||||
.collect();
|
||||
let mut uploaded_parts: Vec<i32> = key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::UploadPart)
|
||||
.filter_map(|record| record.part_number)
|
||||
.collect();
|
||||
uploaded_parts.sort_unstable();
|
||||
uploaded_parts.dedup();
|
||||
let transport = LayoutTransport {
|
||||
status,
|
||||
uploaded_parts,
|
||||
single_puts: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.count(),
|
||||
completes: key_requests
|
||||
.iter()
|
||||
.filter(|record| record.operation == FakeTargetOperation::CompleteMultipartUpload)
|
||||
.count(),
|
||||
journal: key_requests
|
||||
.iter()
|
||||
.map(|record| {
|
||||
(
|
||||
record.sequence,
|
||||
format!("{:?}", record.operation),
|
||||
record.part_number,
|
||||
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
?transport,
|
||||
"replication transport observed on the target"
|
||||
);
|
||||
transports.push(transport);
|
||||
}
|
||||
Ok((target, transports))
|
||||
}
|
||||
|
||||
/// rc.5 writes single-PUT, multipart, compressed, SSE-C and SSE-S3 layouts;
|
||||
/// the current build must read every byte, expose the stored part layout
|
||||
/// through GetObjectAttributes and partNumber reads, and replicate the objects
|
||||
/// with the transport that matches their stored parts.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn direct_upgrade_from_rc5_preserves_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let old_client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
assert_default_sse_s3_encryption(&old_client, LAYOUT_ENCRYPTED_BUCKET, "rc.5").await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&old_client, case).await?;
|
||||
let head = layout_head(&old_client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
case.rc5_reported_parts = layout_attributes(&old_client, case)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|a| layout_reported_parts(&a));
|
||||
tracing::info!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %case.label(),
|
||||
parts = case.part_sizes.len(),
|
||||
etag = %case.rc5_etag,
|
||||
rc5_reported_parts = ?case.rc5_reported_parts,
|
||||
"rc.5 wrote a legacy layout"
|
||||
);
|
||||
}
|
||||
// The rc.5 writer must itself still read what it wrote, so a later
|
||||
// failure is attributable to the upgrade rather than to the fixture.
|
||||
for case in &cases {
|
||||
assert_layout_readable(&old_client, case, "rc.5").await?;
|
||||
}
|
||||
|
||||
// Upgrade in place.
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
for case in &cases {
|
||||
assert_layout_readable(&client, case, "upgraded").await?;
|
||||
assert_layout_attributes(&client, case, "upgraded").await?;
|
||||
}
|
||||
|
||||
// Replicate the pre-existing objects with the current build.
|
||||
let (target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let replica_client = fake_source_client(&target);
|
||||
for (case, transport) in cases.iter().zip(&transports) {
|
||||
let label = case.label();
|
||||
if !case.assert_replication {
|
||||
continue;
|
||||
}
|
||||
assert_eq!(transport.status, "COMPLETED", "{label}: existing-object replication must complete");
|
||||
if case.is_multipart_layout() {
|
||||
let expected: Vec<i32> = (1..=i32::try_from(case.part_sizes.len())?).collect();
|
||||
assert_eq!(
|
||||
transport.uploaded_parts, expected,
|
||||
"{label}: stored parts must replicate as the same multipart layout"
|
||||
);
|
||||
// The current build can drive an existing object twice (two
|
||||
// full CreateMultipartUpload/UploadPart/Complete rounds with
|
||||
// distinct upload ids) while its status is still PENDING; the
|
||||
// rc.5 baseline drives once. That is a scheduling difference,
|
||||
// not a layout one, tracked as rustfs/backlog#2362.
|
||||
assert!(transport.completes >= 1, "{label}: at least one CompleteMultipartUpload");
|
||||
if transport.completes > 1 {
|
||||
tracing::warn!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %label,
|
||||
completes = transport.completes,
|
||||
journal = ?transport.journal,
|
||||
"existing-object replication drove the same object more than once (rustfs/backlog#2362)"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
transport.single_puts, 0,
|
||||
"{label}: a multipart layout must not go out as a single PutObject"
|
||||
);
|
||||
} else {
|
||||
assert!(transport.single_puts >= 1, "{label}: a single PUT replicates as PutObject");
|
||||
assert!(transport.uploaded_parts.is_empty(), "{label}: a single PUT must not go out as multipart");
|
||||
}
|
||||
if !case.ssec {
|
||||
let replica = replica_client
|
||||
.get_object()
|
||||
.bucket(LAYOUT_REPLICA_BUCKET)
|
||||
.key(case.key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("{label}: replica missing on the target: {err}"))?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(replica.len(), case.body.len(), "{label}: replica length");
|
||||
assert!(replica == case.body, "{label}: replica body must equal the rc.5 upload");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The same layouts replicated by rc.5 itself, without an upgrade. This is the
|
||||
/// baseline that tells a pre-existing transport failure apart from one the
|
||||
/// current build introduced; it records the outcome per layout and only fails
|
||||
/// when the fixture cannot run.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn rc5_baseline_replicates_multipart_layouts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&client, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&client, case).await?;
|
||||
let head = layout_head(&client, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: rc.5 HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
}
|
||||
let (_target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let summary: Vec<String> = cases
|
||||
.iter()
|
||||
.zip(&transports)
|
||||
.map(|(case, transport)| {
|
||||
format!(
|
||||
"{}: {} parts={:?} puts={}",
|
||||
case.label(),
|
||||
transport.status,
|
||||
transport.uploaded_parts,
|
||||
transport.single_puts
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
tracing::info!(target: "e2e_test::upgrade_compatibility_test", ?summary, "rc.5 baseline replication outcomes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -118,6 +118,8 @@ hotpath-cpu = [
|
||||
# injection, xl.meta transition assertions) via `api::tier::test_util`.
|
||||
# Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6).
|
||||
test-util = []
|
||||
# Observes real startup CAS only in the dedicated E2E binary.
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
@@ -226,7 +228,7 @@ metrics = { workspace = true }
|
||||
# crates.io. The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.2"
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, VersionIdentityCapability, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -76,26 +76,11 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_disposition {
|
||||
pub use crate::bucket::lifecycle::recovery_disposition::{
|
||||
IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, IlmRecoveryDispositionState,
|
||||
dry_run_recovery_disposition, execute_recovery_disposition,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_export {
|
||||
pub use crate::bucket::lifecycle::recovery_export::{
|
||||
IlmRecoveryExportCreated, IlmRecoveryExportObservation, create_recovery_export,
|
||||
inspect_recovery_export_observation, load_recovery_export,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
|
||||
inspect_transition_transaction_for_operator, retry_transition_recovery_for_operator,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
@@ -308,7 +293,7 @@ pub mod cache {
|
||||
pub mod capacity {
|
||||
pub use crate::core::pools::{
|
||||
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
is_pool_activation_fleet_proof_error, path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
path2_bucket_object, path2_bucket_object_with_base_path,
|
||||
};
|
||||
pub use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
}
|
||||
@@ -383,6 +368,8 @@ pub mod data_usage {
|
||||
pub mod disk {
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
pub use crate::disk::local::ScanGuard;
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
@@ -461,12 +448,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, IlmRecoveryExportFleetProofToken,
|
||||
LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, acquire_ilm_recovery_export_fleet_proof,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_local_process_epoch,
|
||||
ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
@@ -519,8 +503,7 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX,
|
||||
ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
|
||||
TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
|
||||
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
@@ -563,8 +546,8 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
use crate::bucket::target::ARN;
|
||||
use crate::bucket::target::BucketTargetType;
|
||||
@@ -33,8 +33,6 @@ use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_legal_hold::{PutObjectLegalHoldError, PutObjectLegalHoldOutput};
|
||||
use aws_sdk_s3::operation::put_object_retention::{PutObjectRetentionError, PutObjectRetentionOutput};
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -44,7 +42,6 @@ use aws_sdk_s3::types::{
|
||||
ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ServerSideEncryption,
|
||||
};
|
||||
use aws_sdk_s3::types::{ObjectLockLegalHold, ObjectLockRetention};
|
||||
use aws_sdk_s3::{Client as S3Client, operation::head_object::HeadObjectOutput};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
|
||||
use futures::{StreamExt, stream};
|
||||
@@ -129,25 +126,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
}
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
|
||||
/// Whether an edited bucket target still addresses the same remote service
|
||||
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
|
||||
/// about that service stays valid across the edit.
|
||||
fn same_replication_service(edited: &BucketTarget, previous: &BucketTarget) -> bool {
|
||||
let access_key = |target: &BucketTarget| target.credentials.as_ref().map(|credentials| credentials.access_key.clone());
|
||||
edited.endpoint == previous.endpoint
|
||||
&& edited.target_bucket == previous.target_bucket
|
||||
&& edited.secure == previous.secure
|
||||
&& edited.path == previous.path
|
||||
&& access_key(edited) == access_key(previous)
|
||||
}
|
||||
|
||||
/// Page size and page budget for [`TargetClient::locate_replica_by_etag`].
|
||||
const FIND_VERSION_BY_ETAG_PAGE_SIZE: i32 = 1000;
|
||||
const FIND_VERSION_BY_ETAG_MAX_PAGES: usize = 8;
|
||||
/// Candidate cap for [`TargetClient::replica_candidates_by_etag`]: more than
|
||||
/// this many same-content versions of one key is ambiguity by any measure.
|
||||
const FIND_VERSION_BY_ETAG_MAX_MATCHES: usize = 16;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
|
||||
@@ -371,13 +349,6 @@ struct TargetClientBuildProbe {
|
||||
/// their import path while the verdict vocabulary lives with the
|
||||
/// replication decision logic.
|
||||
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||
/// Version-identity verdicts (see the enum's own docs in
|
||||
/// `rustfs-replication`) are cached here per target ARN and follow the same
|
||||
/// `arn_remotes_map` lifecycle. They carry no TTL: the verdict is refreshed
|
||||
/// by every replication write's response, so it can only go stale on a
|
||||
/// target that receives no writes — and a stale `MintsOwn` costs one extra
|
||||
/// content-identity lookup before a PUT, never a lost replica.
|
||||
pub use crate::bucket::replication::VersionIdentityCapability;
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
@@ -404,11 +375,6 @@ pub struct BucketTargetSys {
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
/// Version-identity verdicts keyed by target ARN. See
|
||||
/// [`VersionIdentityCapability`]; reset alongside `arn_remotes_map`. A std
|
||||
/// lock (never held across an await) so the replication worker can record
|
||||
/// a verdict from inside its synchronous PUT-response audit.
|
||||
version_identity_map: Arc<std::sync::RwLock<HashMap<String, VersionIdentityCapability>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
|
||||
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
|
||||
@@ -457,7 +423,6 @@ impl BucketTargetSys {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
version_identity_map: Arc::new(std::sync::RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -781,40 +746,10 @@ impl BucketTargetSys {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached version-identity verdict for a target ARN; `Unknown` until a
|
||||
/// replication write or a replication-check VersionFidelity probe judged
|
||||
/// it since the target was built.
|
||||
pub fn version_identity_capability(&self, arn: &str) -> VersionIdentityCapability {
|
||||
self.version_identity_map
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(arn)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Record a version-identity verdict for a target ARN. Written by the
|
||||
/// replication worker after every PutObject / CompleteMultipartUpload
|
||||
/// response and by the replication-check VersionFidelity phase.
|
||||
pub fn record_version_identity_capability(&self, arn: &str, capability: VersionIdentityCapability) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(arn.to_string(), capability);
|
||||
}
|
||||
|
||||
fn forget_version_identity_capability(&self, arn: &str) {
|
||||
self.version_identity_map
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(arn);
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
@@ -1227,32 +1162,12 @@ impl BucketTargetSys {
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
let unchanged_service: HashMap<&str, &BucketTarget> = targets
|
||||
.map(|new_targets| {
|
||||
new_targets
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| (target.arn.as_str(), target))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
// The version-identity verdict survives an edit that keeps the
|
||||
// same remote service (a resync start or a bandwidth change
|
||||
// rewrites the entry in place): forgetting it there would make
|
||||
// the very resync that follows re-drive every object as a
|
||||
// duplicate on a target that mints its own version ids.
|
||||
if unchanged_service
|
||||
.get(target.arn.as_str())
|
||||
.is_none_or(|edited| !same_replication_service(edited, &target))
|
||||
{
|
||||
self.forget_version_identity_capability(&target.arn);
|
||||
}
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1369,7 +1284,6 @@ fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
|
||||
arn.to_string()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemoveObjectOptions {
|
||||
pub force_delete: bool,
|
||||
pub governance_bypass: bool,
|
||||
@@ -1978,125 +1892,6 @@ impl TargetClient {
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// Candidate replicas by content identity on a target that mints its own
|
||||
/// version ids: page `ListObjectVersions` under the exact key and report
|
||||
/// the live versions whose ETag matches `source_etag`, newest first.
|
||||
/// Delete markers and prefix siblings never match. Bounded to
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_PAGES`] pages and
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_MATCHES`] candidates so a key with a very
|
||||
/// deep history cannot turn one convergence check into an unbounded scan;
|
||||
/// a replica beyond that window reads as missing, which only costs a
|
||||
/// re-PUT (today's behaviour), never a lost object.
|
||||
///
|
||||
/// Content identity is not version identity: two source generations with
|
||||
/// the same bytes have the same ETag. Callers drop the candidates other
|
||||
/// source versions already claim through their ledgers and refuse an
|
||||
/// [`ReplicaLocation::Ambiguous`] remainder before mutating or deleting.
|
||||
pub async fn replica_candidates_by_etag(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_etag: &str,
|
||||
) -> Result<Vec<String>, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
|
||||
let mut key_marker: Option<String> = None;
|
||||
let mut version_id_marker: Option<String> = None;
|
||||
let mut matches: Vec<String> = Vec::new();
|
||||
for _ in 0..FIND_VERSION_BY_ETAG_MAX_PAGES {
|
||||
let page = self
|
||||
.client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(object)
|
||||
.max_keys(FIND_VERSION_BY_ETAG_PAGE_SIZE)
|
||||
.set_key_marker(key_marker.take())
|
||||
.set_version_id_marker(version_id_marker.take())
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)?;
|
||||
matches.extend(
|
||||
page.versions()
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
version.key() == Some(object)
|
||||
&& version.version_id().is_some_and(|id| !id.is_empty())
|
||||
&& replication_etags_match(Some(source_etag), version.e_tag())
|
||||
})
|
||||
.filter_map(|version| version.version_id().map(str::to_string)),
|
||||
);
|
||||
// A listing that moved past the exact key (every listed key is >=
|
||||
// the prefix), ended, or already filled the candidate cap decides.
|
||||
if matches.len() >= FIND_VERSION_BY_ETAG_MAX_MATCHES
|
||||
|| page
|
||||
.versions()
|
||||
.iter()
|
||||
.any(|version| version.key().is_some_and(|key| key > object))
|
||||
|| !page.is_truncated().unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
key_marker = page.next_key_marker().map(str::to_string);
|
||||
version_id_marker = page.next_version_id_marker().map(str::to_string);
|
||||
if key_marker.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
matches.truncate(FIND_VERSION_BY_ETAG_MAX_MATCHES);
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
/// PutObjectRetention against a replica version on a target that does not
|
||||
/// take retention through the replication PUT's own headers (it mints its
|
||||
/// own version ids, so a re-PUT would create another version instead of
|
||||
/// updating this one). Anti-loop marker always added.
|
||||
pub async fn put_object_retention(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: aws_sdk_s3::primitives::DateTime,
|
||||
) -> Result<PutObjectRetentionOutput, Box<SdkError<PutObjectRetentionError>>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.retention(
|
||||
ObjectLockRetention::builder()
|
||||
.mode(mode)
|
||||
.retain_until_date(retain_until)
|
||||
.build(),
|
||||
)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// PutObjectLegalHold counterpart of [`Self::put_object_retention`].
|
||||
pub async fn put_object_legal_hold(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
status: ObjectLockLegalHoldStatus,
|
||||
) -> Result<PutObjectLegalHoldOutput, Box<SdkError<PutObjectLegalHoldError>>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.legal_hold(ObjectLockLegalHold::builder().status(status).build())
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
@@ -2269,15 +2064,7 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
// A forwarded source checksum is this PUT's integrity header. In
|
||||
// streaming-checksum mode (`RUSTFS_REPLICATION_STREAMING_CHECKSUMS`)
|
||||
// the SDK would still add its default CRC32 trailer, and a target that
|
||||
// receives both keeps the trailer's algorithm: a forwarded SHA256
|
||||
// vanished from the replica while the source reported COMPLETED. Pin
|
||||
// this request to WhenRequired so nothing is sent beside the source's
|
||||
// own checksum.
|
||||
let forwards_source_checksum = headers.keys().any(|name| name.as_str().starts_with("x-amz-checksum-"));
|
||||
let mut operation = builder
|
||||
match builder
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.content_length(size)
|
||||
@@ -2297,14 +2084,10 @@ impl TargetClient {
|
||||
}
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
}
|
||||
match operation.send().await {
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
// of the stored plaintext, so it cannot be compared against the
|
||||
@@ -2548,45 +2331,6 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a replica stands on a target that mints its own version ids, by
|
||||
/// content identity (exact key + ETag) after the candidates other source
|
||||
/// versions claim were removed. See
|
||||
/// [`TargetClient::replica_candidates_by_etag`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ReplicaLocation {
|
||||
/// No live version under the key carries the source ETag.
|
||||
Missing,
|
||||
/// Exactly one live version carries it: safe to address.
|
||||
Unique(String),
|
||||
/// More than one live version carries it (same bytes replicated for
|
||||
/// several source generations). `newest` is the most recently listed
|
||||
/// one — good enough to prove the replica exists, never good enough to
|
||||
/// pick which one to mutate or delete.
|
||||
Ambiguous { newest: String },
|
||||
}
|
||||
|
||||
impl ReplicaLocation {
|
||||
/// `matches` newest first, as the target listed them.
|
||||
pub fn from_matches(mut matches: Vec<String>) -> Self {
|
||||
match matches.len() {
|
||||
0 => Self::Missing,
|
||||
1 => Self::Unique(matches.remove(0)),
|
||||
_ => Self::Ambiguous {
|
||||
newest: matches.remove(0),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The version to read for existence/ETag checks, where an ambiguous
|
||||
/// match is still a located replica.
|
||||
pub fn any_version_id(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Missing => None,
|
||||
Self::Unique(version_id) | Self::Ambiguous { newest: version_id } => Some(version_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum BucketTargetError {
|
||||
BucketRemoteTargetNotFound {
|
||||
@@ -2763,21 +2507,13 @@ mod tests {
|
||||
}
|
||||
|
||||
fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
|
||||
header_recording_target_client_with_checksums(response_headers, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn header_recording_target_client_with_checksums(
|
||||
response_headers: Vec<(String, String)>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> (TargetClient, RecordedHeaders) {
|
||||
let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
response_headers,
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client =
|
||||
s3_client_for_endpoint_test_with_checksums("https://localhost:443".to_string(), Some(http_client), checksums);
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
@@ -2944,47 +2680,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// With streaming checksums enabled the SDK adds a CRC32 trailer to every
|
||||
/// upload. A PUT that forwards the source's checksum must not get that
|
||||
/// second algorithm: a target that receives both keeps the trailer's and
|
||||
/// the forwarded SHA256 never reaches the replica (rustfs/backlog#2340).
|
||||
#[tokio::test]
|
||||
async fn streaming_put_object_with_forwarded_checksum_sends_no_sdk_checksum() {
|
||||
let (client, recorded) =
|
||||
header_recording_target_client_with_checksums(Vec::new(), RequestChecksumCalculation::WhenSupported);
|
||||
let mut forwarded = PutObjectOptions::default();
|
||||
forwarded.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_string(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_string(),
|
||||
);
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &forwarded)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
|
||||
let with_forwarded = &recorded[0];
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-checksum-sha256"),
|
||||
Some("OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=")
|
||||
);
|
||||
assert_eq!(
|
||||
recorded_header(with_forwarded, "x-amz-trailer"),
|
||||
None,
|
||||
"the SDK must not add a trailer checksum"
|
||||
);
|
||||
assert_eq!(recorded_header(with_forwarded, "x-amz-sdk-checksum-algorithm"), None);
|
||||
// Control: the same client still streams a trailer when nothing is forwarded.
|
||||
let without_forwarded = &recorded[1];
|
||||
assert!(
|
||||
recorded_header(without_forwarded, "x-amz-trailer").is_some(),
|
||||
"streaming mode must still apply to uploads without a forwarded checksum: {without_forwarded:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A forwarded source checksum already satisfies the rule; nothing is added.
|
||||
#[tokio::test]
|
||||
async fn locked_put_object_keeps_a_forwarded_source_checksum() {
|
||||
@@ -3350,14 +3045,6 @@ mod tests {
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test(endpoint: String, http_client: Option<SharedHttpClient>) -> S3Client {
|
||||
s3_client_for_endpoint_test_with_checksums(endpoint, http_client, replication_request_checksum_calculation())
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test_with_checksums(
|
||||
endpoint: String,
|
||||
http_client: Option<SharedHttpClient>,
|
||||
checksums: RequestChecksumCalculation,
|
||||
) -> S3Client {
|
||||
let credentials = SdkCredentials::builder()
|
||||
.access_key_id("test-access")
|
||||
.secret_access_key("test-secret")
|
||||
@@ -3371,7 +3058,7 @@ mod tests {
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
// Mirror the production remote-target builder so recorded requests
|
||||
// exercise the same checksum/framing behavior (#6853).
|
||||
.request_checksum_calculation(checksums);
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
@@ -3534,64 +3221,6 @@ mod tests {
|
||||
assert!(message.contains("connection refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_replication_service_ignores_resync_and_bandwidth_edits() {
|
||||
let base = BucketTarget {
|
||||
endpoint: "target.example:9000".to_string(),
|
||||
target_bucket: "replica".to_string(),
|
||||
secure: true,
|
||||
path: "on".to_string(),
|
||||
arn: "arn:rustfs:replication:us-east-1:bucket:same".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let resync_edit = BucketTarget {
|
||||
reset_id: "reset-1".to_string(),
|
||||
bandwidth_limit: 1024,
|
||||
..base.clone()
|
||||
};
|
||||
assert!(same_replication_service(&resync_edit, &base));
|
||||
for moved in [
|
||||
BucketTarget {
|
||||
endpoint: "other.example:9000".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
target_bucket: "other".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
secure: false,
|
||||
..base.clone()
|
||||
},
|
||||
BucketTarget {
|
||||
credentials: Some(Credentials {
|
||||
access_key: "rotated".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..base.clone()
|
||||
},
|
||||
] {
|
||||
assert!(!same_replication_service(&moved, &base));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_verdict_is_per_arn_and_forgotten_with_the_target() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:identity";
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
sys.record_version_identity_capability(arn, VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::MintsOwn);
|
||||
assert_eq!(sys.version_identity_capability("other"), VersionIdentityCapability::Unknown);
|
||||
// A rebuilt target may point at a different service.
|
||||
sys.forget_version_identity_capability(arn);
|
||||
assert_eq!(sys.version_identity_capability(arn), VersionIdentityCapability::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_health_key_preserves_explicit_port() {
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::bucket::lifecycle::manual_transition_job::{
|
||||
record_manual_transition_worker_result_with_reason, renew_manual_transition_job_lease_if_owned,
|
||||
save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, update_manual_transition_job_record,
|
||||
};
|
||||
use crate::bucket::lifecycle::recovery_disposition_runtime::run_recovery_disposition_maintenance_loop;
|
||||
use crate::bucket::lifecycle::replication_sink;
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, ReplicationStatusType, replication_state_to_filemeta,
|
||||
@@ -150,7 +149,6 @@ pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
|
||||
static XXHASH_SEED: u64 = 0;
|
||||
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static RECOVERY_DISPOSITION_MAINTENANCE_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
@@ -2400,20 +2398,9 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
|
||||
spawn_tier_delete_journal_recovery_once(api.clone());
|
||||
spawn_transition_transaction_recovery_once(api.clone());
|
||||
spawn_recovery_disposition_maintenance_once(api.clone());
|
||||
spawn_manual_transition_job_recovery_once(api);
|
||||
}
|
||||
|
||||
fn spawn_recovery_disposition_maintenance_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
let cancel_token = api.ctx.background_cancel_token()?;
|
||||
if RECOVERY_DISPOSITION_MAINTENANCE_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
run_recovery_disposition_maintenance_loop(api, cancel_token).await;
|
||||
}))
|
||||
}
|
||||
|
||||
fn spawn_manual_transition_job_recovery_once(api: Arc<ECStore>) -> Option<JoinHandle<()>> {
|
||||
if MANUAL_TRANSITION_JOB_RECOVERY_STARTED.set(()).is_err() {
|
||||
return None;
|
||||
|
||||
@@ -41,21 +41,6 @@ where
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -71,26 +56,6 @@ where
|
||||
com::read_config_with_metadata(api, file, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -161,30 +126,20 @@ where
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
delete_config_if_match_with_opts(api, file, etag, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_config_if_match_with_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
etag: &str,
|
||||
mut options: ObjectOptions,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
options.http_preconditions = Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
match api.delete_object(RUSTFS_META_BUCKET, file, options).await {
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
|
||||
},
|
||||
manual_transition_job, recovery_control, recovery_disposition, recovery_export, tier_delete_journal, transition_transaction,
|
||||
manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
@@ -42,8 +42,6 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
ManualTransitionTask,
|
||||
ManualTransitionWorkerResult,
|
||||
RecoveryControl,
|
||||
RecoveryExport,
|
||||
RecoveryDisposition,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -114,20 +112,8 @@ pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNam
|
||||
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryControl,
|
||||
};
|
||||
pub(crate) const RECOVERY_EXPORT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-export",
|
||||
prefix: recovery_export::ILM_RECOVERY_EXPORT_PREFIX,
|
||||
max_record_size: recovery_export::MAX_ILM_RECOVERY_EXPORT_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryExport,
|
||||
};
|
||||
pub(crate) const RECOVERY_DISPOSITION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-disposition",
|
||||
prefix: recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
max_record_size: recovery_disposition::MAX_ILM_RECOVERY_DISPOSITION_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryDisposition,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
@@ -138,8 +124,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
RECOVERY_CONTROL_NAMESPACE,
|
||||
RECOVERY_EXPORT_NAMESPACE,
|
||||
RECOVERY_DISPOSITION_NAMESPACE,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -277,28 +261,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
owner_fence_sha256: Option<String>,
|
||||
},
|
||||
RecoveryExport {
|
||||
content_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
topology_generation: String,
|
||||
member_epochs_sha256: String,
|
||||
creator_sha256: String,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
RecoveryDisposition {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
copy_manifest_sha256: String,
|
||||
copy_manifest_count: usize,
|
||||
created_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence_sha256: Option<String>,
|
||||
owner_lease_acquired_at_unix_nanos: Option<i64>,
|
||||
owner_lease_expires_at_unix_nanos: Option<i64>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
retain_until_unix_nanos: i64,
|
||||
},
|
||||
}
|
||||
|
||||
impl DurableIlmRecordCheckpoint {
|
||||
@@ -313,9 +275,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 }
|
||||
| Self::RecoveryControl { content_sha256, .. }
|
||||
| Self::RecoveryExport { content_sha256, .. }
|
||||
| Self::RecoveryDisposition { content_sha256, .. } => content_sha256,
|
||||
| Self::RecoveryControl { content_sha256, .. } => content_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -356,9 +316,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
|
||||
}
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return Err(Error::other("durable ILM recovery disposition checkpoint is invalid"));
|
||||
}
|
||||
}
|
||||
if self == next {
|
||||
if let Self::ManualTransitionJob {
|
||||
@@ -499,7 +456,9 @@ impl DurableIlmRecordCheckpoint {
|
||||
},
|
||||
) => {
|
||||
previous_identity == next_identity
|
||||
&& transition_state_revision_is_successor(*previous_state, *previous_revision, *next_state, *next_revision)
|
||||
&& transition_state_distance(*previous_state, *next_state)
|
||||
.and_then(|distance| previous_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| *next_revision == expected_revision)
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
@@ -635,83 +594,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
&& previous_attempts == next_attempts;
|
||||
adjacent && (claim || source_refresh || completion)
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
owner_lease_acquired_at_unix_nanos: previous_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: previous_owner_expires,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: next_identity,
|
||||
copy_manifest_sha256: next_manifest,
|
||||
copy_manifest_count: next_manifest_count,
|
||||
created_at_unix_nanos: next_created_at,
|
||||
revision: next_revision,
|
||||
state: next_state,
|
||||
owner_fence_sha256: next_owner,
|
||||
owner_lease_acquired_at_unix_nanos: next_owner_acquired,
|
||||
owner_lease_expires_at_unix_nanos: next_owner_expires,
|
||||
confirmed_absent_sha256: next_confirmed,
|
||||
retain_until_unix_nanos: next_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let immutable_identity_matches = previous_identity == next_identity
|
||||
&& previous_manifest == next_manifest
|
||||
&& previous_manifest_count == next_manifest_count
|
||||
&& previous_created_at == next_created_at
|
||||
&& previous_retain_until == next_retain_until;
|
||||
let adjacent = previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let progress_is_monotonic = sorted_sha256_set_is_subset(previous_confirmed, next_confirmed);
|
||||
let legal_edge = match (previous_state, next_state) {
|
||||
(Prepared, Prepared) => {
|
||||
let claim = previous_owner.is_none() && next_owner.is_some();
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
previous_confirmed == next_confirmed && (claim || takeover)
|
||||
}
|
||||
(Prepared, Applying) => {
|
||||
previous_confirmed == next_confirmed
|
||||
&& previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
}
|
||||
(Applying, Applying) => {
|
||||
let progress = previous_owner == next_owner
|
||||
&& previous_owner_acquired == next_owner_acquired
|
||||
&& previous_owner_expires == next_owner_expires
|
||||
&& previous_confirmed.len().checked_add(1) == Some(next_confirmed.len());
|
||||
let takeover = previous_owner.is_some()
|
||||
&& previous_owner != next_owner
|
||||
&& previous_confirmed == next_confirmed
|
||||
&& previous_owner_expires
|
||||
.zip(*next_owner_acquired)
|
||||
.is_some_and(|(expires, acquired)| acquired >= expires);
|
||||
progress || takeover
|
||||
}
|
||||
(Applying, Completed) => {
|
||||
previous_owner.is_some() && next_owner.is_none() && previous_confirmed == next_confirmed
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
immutable_identity_matches && adjacent && progress_is_monotonic && legal_edge
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
@@ -729,11 +611,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
for checkpoint in [self, terminal] {
|
||||
if !recovery_disposition_checkpoint_is_valid(checkpoint) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if let Self::TierProbeIntent { state, .. } = terminal
|
||||
&& !matches!(
|
||||
state,
|
||||
@@ -750,11 +627,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryDisposition { state, .. } = terminal
|
||||
&& state != &recovery_disposition::IlmRecoveryDispositionState::Completed
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -880,121 +752,11 @@ impl DurableIlmRecordCheckpoint {
|
||||
&& terminal_revision > previous_revision
|
||||
&& terminal_attempts >= previous_attempts
|
||||
}
|
||||
(
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: previous_identity,
|
||||
copy_manifest_sha256: previous_manifest,
|
||||
copy_manifest_count: previous_manifest_count,
|
||||
created_at_unix_nanos: previous_created_at,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
owner_fence_sha256: previous_owner,
|
||||
confirmed_absent_sha256: previous_confirmed,
|
||||
retain_until_unix_nanos: previous_retain_until,
|
||||
..
|
||||
},
|
||||
Self::RecoveryDisposition {
|
||||
identity_sha256: terminal_identity,
|
||||
copy_manifest_sha256: terminal_manifest,
|
||||
copy_manifest_count: terminal_manifest_count,
|
||||
created_at_unix_nanos: terminal_created_at,
|
||||
revision: terminal_revision,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState::Completed,
|
||||
confirmed_absent_sha256: terminal_confirmed,
|
||||
retain_until_unix_nanos: terminal_retain_until,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
matches!(
|
||||
previous_state,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared
|
||||
| recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
) && previous_identity == terminal_identity
|
||||
&& previous_manifest == terminal_manifest
|
||||
&& previous_manifest_count == terminal_manifest_count
|
||||
&& previous_created_at == terminal_created_at
|
||||
&& previous_retain_until == terminal_retain_until
|
||||
&& terminal_revision.checked_sub(*previous_revision).is_some_and(|distance| {
|
||||
let minimum_distance = match previous_state {
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared if previous_owner.is_some() => 3,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Prepared => 4,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying
|
||||
if previous_confirmed.len() == *previous_manifest_count =>
|
||||
{
|
||||
1
|
||||
}
|
||||
recovery_disposition::IlmRecoveryDispositionState::Applying => 2,
|
||||
recovery_disposition::IlmRecoveryDispositionState::Completed => u64::MAX,
|
||||
};
|
||||
distance >= minimum_distance
|
||||
})
|
||||
&& sorted_sha256_set_is_subset(previous_confirmed, terminal_confirmed)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint_is_valid(checkpoint: &DurableIlmRecordCheckpoint) -> bool {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
copy_manifest_sha256,
|
||||
copy_manifest_count,
|
||||
created_at_unix_nanos,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos,
|
||||
} = checkpoint
|
||||
else {
|
||||
return true;
|
||||
};
|
||||
let owner_fence_sha256 = owner_fence_sha256.as_deref();
|
||||
|
||||
is_canonical_sha256(content_sha256)
|
||||
&& is_canonical_sha256(identity_sha256)
|
||||
&& is_canonical_sha256(copy_manifest_sha256)
|
||||
&& *copy_manifest_count > 0
|
||||
&& *created_at_unix_nanos > 0
|
||||
&& *revision > 0
|
||||
&& *retain_until_unix_nanos > 0
|
||||
&& owner_fence_sha256.is_none_or(is_canonical_sha256)
|
||||
&& match (
|
||||
owner_fence_sha256,
|
||||
*owner_lease_acquired_at_unix_nanos,
|
||||
*owner_lease_expires_at_unix_nanos,
|
||||
) {
|
||||
(None, None, None) => true,
|
||||
(Some(_), Some(acquired), Some(expires)) => acquired > 0 && expires > acquired,
|
||||
_ => false,
|
||||
}
|
||||
&& confirmed_absent_sha256.len() <= *copy_manifest_count
|
||||
&& confirmed_absent_sha256.iter().all(|digest| is_canonical_sha256(digest))
|
||||
&& confirmed_absent_sha256.windows(2).all(|pair| pair[0] < pair[1])
|
||||
&& match *state {
|
||||
Prepared => confirmed_absent_sha256.is_empty(),
|
||||
Applying => owner_fence_sha256.is_some(),
|
||||
Completed => owner_fence_sha256.is_none() && confirmed_absent_sha256.len() == *copy_manifest_count,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_canonical_sha256(value: &str) -> bool {
|
||||
is_sha256_checksum(value)
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn sorted_sha256_set_is_subset(subset: &[String], superset: &[String]) -> bool {
|
||||
subset.iter().all(|candidate| superset.binary_search(candidate).is_ok())
|
||||
}
|
||||
|
||||
fn tier_delete_dispatch_parent_progress_delta(
|
||||
previous_sequence: u64,
|
||||
previous_completed_journals: u64,
|
||||
@@ -1028,22 +790,6 @@ fn transition_state_distance(
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_state_revision_is_successor(
|
||||
from: transition_transaction::TransitionTransactionState,
|
||||
from_revision: u64,
|
||||
to: transition_transaction::TransitionTransactionState,
|
||||
to_revision: u64,
|
||||
) -> bool {
|
||||
use transition_transaction::TransitionTransactionState::{LocalCommitStarted, UploadOutcomeUnknown};
|
||||
|
||||
if from == UploadOutcomeUnknown && from_revision == 1 && to == LocalCommitStarted {
|
||||
return to_revision == 2;
|
||||
}
|
||||
transition_state_distance(from, to)
|
||||
.and_then(|distance| from_revision.checked_add(distance))
|
||||
.is_some_and(|expected_revision| to_revision == expected_revision)
|
||||
}
|
||||
|
||||
fn tier_probe_state_reaches(
|
||||
from: tier_probe_intent::TierProbeIntentState,
|
||||
to: tier_probe_intent::TierProbeIntentState,
|
||||
@@ -1602,55 +1348,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryExport => {
|
||||
let (protocol, export_id) = recovery_export::recovery_export_id_from_record_object_name(path)?;
|
||||
let export = recovery_export::IlmRecoveryExport::decode(&export_id, data)?;
|
||||
let canonical = recovery_export::recovery_export_record_object_name(protocol, &export_id)?;
|
||||
if canonical != path || export.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
let source_generation_sha256 = checkpoint_hash(&export.source_generation)?;
|
||||
(
|
||||
"export_id",
|
||||
export_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryExport {
|
||||
content_sha256,
|
||||
source_generation_sha256,
|
||||
topology_generation: export.topology_generation,
|
||||
member_epochs_sha256: export.member_epochs_sha256,
|
||||
creator_sha256: export.creator_sha256,
|
||||
retain_until_unix_nanos: export.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryDisposition => {
|
||||
// The disposition module owns strict schema, checksum, canonical
|
||||
// path, immutable-manifest, and state-specific validation. Keep
|
||||
// this boundary limited to decommission identity/checkpoint
|
||||
// projection so the two readers cannot accept different records.
|
||||
let disposition = recovery_disposition::decode_recovery_disposition_checkpoint(path, data)?;
|
||||
if disposition.content_sha256 != content_sha256 {
|
||||
return Err(Error::other("ILM recovery disposition checkpoint content digest is invalid"));
|
||||
}
|
||||
(
|
||||
"disposition_id",
|
||||
disposition.disposition_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: disposition.content_sha256,
|
||||
identity_sha256: disposition.identity_sha256,
|
||||
copy_manifest_sha256: disposition.copy_manifest_sha256,
|
||||
copy_manifest_count: disposition.copy_manifest_count,
|
||||
created_at_unix_nanos: disposition.created_at_unix_nanos,
|
||||
revision: disposition.revision,
|
||||
state: disposition.state,
|
||||
owner_fence_sha256: disposition.owner_fence_sha256,
|
||||
owner_lease_acquired_at_unix_nanos: disposition.owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos: disposition.owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256: disposition.confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: disposition.retain_until_unix_nanos,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1806,203 +1503,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_disposition_checkpoint(
|
||||
revision: u64,
|
||||
state: recovery_disposition::IlmRecoveryDispositionState,
|
||||
owner_fence: Option<&str>,
|
||||
confirmed_absent_sha256: Vec<String>,
|
||||
) -> DurableIlmRecordCheckpoint {
|
||||
let (owner_lease_acquired_at_unix_nanos, owner_lease_expires_at_unix_nanos) = match owner_fence {
|
||||
Some("f") => (Some(10), Some(20)),
|
||||
Some(_) => (Some(1), Some(10)),
|
||||
None => (None, None),
|
||||
};
|
||||
DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: "a".repeat(64),
|
||||
copy_manifest_sha256: "d".repeat(64),
|
||||
copy_manifest_count: 2,
|
||||
created_at_unix_nanos: 1_700_000_000_000_000_000,
|
||||
revision,
|
||||
state,
|
||||
owner_fence_sha256: owner_fence.map(|digest| digest.repeat(64)),
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
owner_lease_expires_at_unix_nanos,
|
||||
confirmed_absent_sha256,
|
||||
retain_until_unix_nanos: 1_820_000_000_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_namespace_is_registered_without_shadowing_its_root() {
|
||||
let disposition_id = "a".repeat(64);
|
||||
let path = format!(
|
||||
"{}/tier_delete_journal/{}/{}/{}.json",
|
||||
recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX,
|
||||
&disposition_id[..2],
|
||||
&disposition_id[2..4],
|
||||
disposition_id
|
||||
);
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery disposition path should classify")
|
||||
.expect("recovery disposition should be durable");
|
||||
|
||||
assert_eq!(namespace, &RECOVERY_DISPOSITION_NAMESPACE);
|
||||
assert!(classify_durable_ilm_record(recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_checkpoint_accepts_only_monotonic_progress() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let claimed = recovery_disposition_checkpoint(2, Prepared, Some("e"), Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), Vec::new());
|
||||
let first_absent = recovery_disposition_checkpoint(4, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let taken_over = recovery_disposition_checkpoint(5, Applying, Some("f"), vec![first_copy.clone()]);
|
||||
let all_absent = recovery_disposition_checkpoint(6, Applying, Some("f"), vec![first_copy.clone(), second_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(7, Completed, None, vec![first_copy.clone(), second_copy.clone()]);
|
||||
|
||||
prepared
|
||||
.validate_successor(&claimed)
|
||||
.expect("Prepared should record an owner claim without absence progress");
|
||||
claimed
|
||||
.validate_successor(&applying)
|
||||
.expect("Prepared should advance to Applying without folding in deletion progress");
|
||||
applying
|
||||
.validate_successor(&first_absent)
|
||||
.expect("Applying should append newly confirmed absent copies");
|
||||
first_absent
|
||||
.validate_successor(&taken_over)
|
||||
.expect("Applying should record a fenced owner takeover without losing progress");
|
||||
taken_over
|
||||
.validate_successor(&all_absent)
|
||||
.expect("Applying should preserve every earlier confirmation while making progress");
|
||||
all_absent
|
||||
.validate_successor(&completed)
|
||||
.expect("a fully confirmed manifest should advance to Completed");
|
||||
|
||||
assert!(
|
||||
prepared.validate_successor(&completed).is_err(),
|
||||
"adjacent receipt updates must not skip Applying"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), Vec::new()))
|
||||
.is_err(),
|
||||
"confirmed-absent progress must not move backwards"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Completed, None, vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"Completed must cover the complete immutable copy manifest"
|
||||
);
|
||||
assert!(
|
||||
completed
|
||||
.validate_successor(&recovery_disposition_checkpoint(7, Applying, Some("e"), vec![second_copy]))
|
||||
.is_err(),
|
||||
"Completed is terminal"
|
||||
);
|
||||
assert!(
|
||||
first_absent
|
||||
.validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), vec![first_copy.clone()]))
|
||||
.is_err(),
|
||||
"a same-state revision bump must change the owner fence or absence progress"
|
||||
);
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(4, Applying, None, vec![first_copy]))
|
||||
.is_err(),
|
||||
"Applying must retain a fenced owner"
|
||||
);
|
||||
|
||||
let mut noncanonical_identity = claimed.clone();
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut noncanonical_identity {
|
||||
*identity_sha256 = "A".repeat(64);
|
||||
}
|
||||
assert!(prepared.validate_successor(&noncanonical_identity).is_err());
|
||||
let mut changed_created_at = claimed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
created_at_unix_nanos, ..
|
||||
} = &mut changed_created_at
|
||||
{
|
||||
*created_at_unix_nanos += 1;
|
||||
}
|
||||
assert!(prepared.validate_successor(&changed_created_at).is_err());
|
||||
|
||||
let mut early_takeover = taken_over;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition {
|
||||
owner_lease_acquired_at_unix_nanos,
|
||||
..
|
||||
} = &mut early_takeover
|
||||
{
|
||||
*owner_lease_acquired_at_unix_nanos = Some(9);
|
||||
}
|
||||
assert!(first_absent.validate_successor(&early_takeover).is_err());
|
||||
assert!(
|
||||
applying
|
||||
.validate_successor(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Applying,
|
||||
Some("e"),
|
||||
vec!["c".repeat(64), "b".repeat(64)],
|
||||
))
|
||||
.is_err(),
|
||||
"confirmed-absent entries must be a canonical sorted set"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_disposition_terminal_predecessor_requires_exact_identity_and_full_manifest() {
|
||||
use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared};
|
||||
|
||||
let first_copy = "b".repeat(64);
|
||||
let second_copy = "c".repeat(64);
|
||||
let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new());
|
||||
let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), vec![first_copy.clone()]);
|
||||
let completed = recovery_disposition_checkpoint(5, Completed, None, vec![first_copy.clone(), second_copy]);
|
||||
|
||||
assert!(prepared.is_predecessor_of_terminal(&completed));
|
||||
assert!(applying.is_predecessor_of_terminal(&completed));
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(2, Applying, Some("e"), Vec::new())),
|
||||
"a nonterminal disposition must not authorize terminal cleanup"
|
||||
);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"terminal proof must leave enough revisions for claim, apply, progress, and completion"
|
||||
);
|
||||
assert!(
|
||||
!applying.is_predecessor_of_terminal(&recovery_disposition_checkpoint(
|
||||
4,
|
||||
Completed,
|
||||
None,
|
||||
vec![first_copy.clone(), "c".repeat(64)],
|
||||
)),
|
||||
"an incomplete Applying checkpoint cannot complete without a progress generation"
|
||||
);
|
||||
|
||||
let mut other_identity = completed;
|
||||
if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut other_identity {
|
||||
*identity_sha256 = "e".repeat(64);
|
||||
}
|
||||
assert!(!prepared.is_predecessor_of_terminal(&other_identity));
|
||||
|
||||
let incomplete_terminal = recovery_disposition_checkpoint(4, Completed, None, vec![first_copy]);
|
||||
assert!(
|
||||
!prepared.is_predecessor_of_terminal(&incomplete_terminal),
|
||||
"a partial confirmed-absent set must not become terminal proof"
|
||||
);
|
||||
}
|
||||
|
||||
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
|
||||
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
tier_probe_intent::TierProbeIntent {
|
||||
@@ -2187,64 +1687,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_checkpoint_accepts_only_the_distinguishable_compact_edge() {
|
||||
let identity_sha256 = "a".repeat(64);
|
||||
let unknown_remote_sha256 = "b".repeat(64);
|
||||
let known_remote_sha256 = "c".repeat(64);
|
||||
let checkpoint = |revision, state, remote_version_sha256: String, remote_version_known| {
|
||||
DurableIlmRecordCheckpoint::TransitionTransaction {
|
||||
content_sha256: format!("{revision:064x}"),
|
||||
identity_sha256: identity_sha256.clone(),
|
||||
remote_version_sha256,
|
||||
remote_version_known,
|
||||
revision,
|
||||
state,
|
||||
}
|
||||
};
|
||||
let compact_unknown = checkpoint(
|
||||
1,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256.clone(),
|
||||
false,
|
||||
);
|
||||
let compact_local_commit = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
compact_unknown
|
||||
.validate_successor(&compact_local_commit)
|
||||
.expect("compact pre-upload fence should advance directly to the exact local-commit fence");
|
||||
|
||||
let legacy_unknown = checkpoint(
|
||||
2,
|
||||
transition_transaction::TransitionTransactionState::UploadOutcomeUnknown,
|
||||
unknown_remote_sha256,
|
||||
false,
|
||||
);
|
||||
let invalid_legacy_skip = checkpoint(
|
||||
3,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256.clone(),
|
||||
true,
|
||||
);
|
||||
assert!(
|
||||
legacy_unknown.validate_successor(&invalid_legacy_skip).is_err(),
|
||||
"legacy UploadOutcomeUnknown@2 must not masquerade as the compact edge"
|
||||
);
|
||||
let valid_legacy_skip = checkpoint(
|
||||
4,
|
||||
transition_transaction::TransitionTransactionState::LocalCommitStarted,
|
||||
known_remote_sha256,
|
||||
true,
|
||||
);
|
||||
legacy_unknown
|
||||
.validate_successor(&valid_legacy_skip)
|
||||
.expect("legacy receipts may still observe the existing two-edge state advance");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
@@ -25,9 +25,6 @@ mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod recovery_control;
|
||||
pub mod recovery_disposition;
|
||||
pub(crate) mod recovery_disposition_runtime;
|
||||
pub mod recovery_export;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
|
||||
@@ -168,7 +168,7 @@ impl IlmRecoverySourceGeneration {
|
||||
Ok(generation)
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
fn validate(&self) -> Result<()> {
|
||||
if self.source_schema.trim().is_empty() {
|
||||
return Err(IlmRecoveryControlError::Corrupt("source schema is empty"));
|
||||
}
|
||||
@@ -485,40 +485,6 @@ impl IlmRecoveryControl {
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn abandon_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> {
|
||||
if self.owner.is_some()
|
||||
|| self.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| &self.observed_source_generation != expected_source_generation
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator abandonment requires the exact ownerless retained source generation",
|
||||
));
|
||||
}
|
||||
self.bump_revision()?;
|
||||
self.classification = IlmRecoveryClassification::Abandoned;
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn retry_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> {
|
||||
if self.owner.is_some()
|
||||
|| !matches!(
|
||||
self.classification,
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired
|
||||
)
|
||||
|| self.attempt_count == u64::MAX
|
||||
|| &self.observed_source_generation != expected_source_generation
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator retry requires the exact ownerless retained source generation",
|
||||
));
|
||||
}
|
||||
self.bump_revision()?;
|
||||
self.classification = IlmRecoveryClassification::Retrying;
|
||||
self.consecutive_failure_count = 0;
|
||||
self.next_attempt_at_unix_nanos = None;
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub fn validate_successor(&self, next: &Self) -> Result<()> {
|
||||
self.validate()?;
|
||||
next.validate()?;
|
||||
@@ -542,58 +508,12 @@ impl IlmRecoveryControl {
|
||||
self.validate_failure_successor(next)
|
||||
}
|
||||
(Some(_), None) => self.validate_finish_successor(next),
|
||||
(None, None)
|
||||
if self.classification == IlmRecoveryClassification::RetainedAmbiguous
|
||||
&& next.classification == IlmRecoveryClassification::Abandoned =>
|
||||
{
|
||||
self.validate_operator_abandon_successor(next)
|
||||
}
|
||||
(None, None)
|
||||
if matches!(
|
||||
self.classification,
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired
|
||||
) && next.classification == IlmRecoveryClassification::Retrying =>
|
||||
{
|
||||
self.validate_operator_retry_successor(next)
|
||||
}
|
||||
(None, None) => Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"ownerless control cannot advance without a claim",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_operator_abandon_successor(&self, next: &Self) -> Result<()> {
|
||||
if next.observed_source_generation != self.observed_source_generation
|
||||
|| next.attempt_count != self.attempt_count
|
||||
|| next.consecutive_failure_count != self.consecutive_failure_count
|
||||
|| next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos
|
||||
|| next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos
|
||||
|| next.next_attempt_at_unix_nanos != self.next_attempt_at_unix_nanos
|
||||
|| next.last_error_code != self.last_error_code
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator abandonment changed recovery history or source generation",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_operator_retry_successor(&self, next: &Self) -> Result<()> {
|
||||
if next.observed_source_generation != self.observed_source_generation
|
||||
|| next.attempt_count != self.attempt_count
|
||||
|| next.consecutive_failure_count != 0
|
||||
|| next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos
|
||||
|| next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos
|
||||
|| next.next_attempt_at_unix_nanos.is_some()
|
||||
|| next.last_error_code != self.last_error_code
|
||||
{
|
||||
return Err(IlmRecoveryControlError::InvalidSuccessor(
|
||||
"operator retry changed recovery history or source generation",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_claim_successor(&self, next: &Self) -> Result<()> {
|
||||
if self.classification != IlmRecoveryClassification::Retrying
|
||||
|| next.classification != IlmRecoveryClassification::Retrying
|
||||
@@ -907,23 +827,6 @@ pub async fn observe_recovery_source(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
observe_recovery_source_with_options(api, canonical_path, source_schema, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn observe_recovery_source_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
observe_recovery_source_with_options(api, canonical_path, source_schema, true).await
|
||||
}
|
||||
|
||||
async fn observe_recovery_source_with_options(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
no_lock: bool,
|
||||
) -> EcstoreResult<ObservedIlmRecoverySource> {
|
||||
validate_canonical_source_path(canonical_path).map_err(recovery_control_store_error)?;
|
||||
if source_schema.trim().is_empty() {
|
||||
@@ -934,16 +837,7 @@ async fn observe_recovery_source_with_options(
|
||||
let mut observations = Vec::new();
|
||||
for set in api.all_set_disks() {
|
||||
let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index);
|
||||
match config_boundary::read_config_with_metadata(
|
||||
set,
|
||||
canonical_path,
|
||||
&ObjectOptions {
|
||||
no_lock,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
match config_boundary::read_config_with_metadata(set, canonical_path, &ObjectOptions::default()).await {
|
||||
Ok((data, metadata)) => {
|
||||
let etag = metadata
|
||||
.etag
|
||||
@@ -1361,99 +1255,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_abandonment_is_an_exact_ownerless_retained_successor() {
|
||||
let mut retained = IlmRecoveryControl::new(
|
||||
control().identity,
|
||||
generation(),
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
1_000_000_000,
|
||||
IlmRecoveryErrorCode::OperatorDispositionRequired,
|
||||
)
|
||||
.expect("retained control should build");
|
||||
let previous = retained.clone();
|
||||
retained
|
||||
.abandon_for_operator(&previous.observed_source_generation)
|
||||
.expect("exact retained generation should be abandonable");
|
||||
previous
|
||||
.validate_successor(&retained)
|
||||
.expect("operator abandonment should be a valid successor");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::Abandoned);
|
||||
assert_eq!(retained.revision, previous.revision + 1);
|
||||
|
||||
let mut wrong_generation = previous.clone();
|
||||
let mut generation = previous.observed_source_generation.clone();
|
||||
generation.source_etag = "different".to_string();
|
||||
assert!(wrong_generation.abandon_for_operator(&generation).is_err());
|
||||
|
||||
let mut mutated_history = retained.clone();
|
||||
mutated_history.attempt_count += 1;
|
||||
assert!(previous.validate_successor(&mutated_history).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_retry_rearms_exact_retained_generation_without_resetting_history() {
|
||||
for classification in [
|
||||
IlmRecoveryClassification::RetainedAmbiguous,
|
||||
IlmRecoveryClassification::OperatorRequired,
|
||||
] {
|
||||
let mut retained = control();
|
||||
retained
|
||||
.claim("node-a", Uuid::new_v4(), 2_000_000_000, 1)
|
||||
.expect("attempt should claim");
|
||||
retained
|
||||
.record_retryable_failure(2_000_000_001, IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("failure should persist");
|
||||
retained.classification = classification;
|
||||
retained.next_attempt_at_unix_nanos = None;
|
||||
if classification == IlmRecoveryClassification::OperatorRequired {
|
||||
retained.attempt_count = u64::from(MAX_RECOVERY_ATTEMPTS);
|
||||
retained.consecutive_failure_count = MAX_RECOVERY_ATTEMPTS;
|
||||
}
|
||||
retained.validate().expect("retained control should remain valid");
|
||||
|
||||
let previous = retained.clone();
|
||||
retained
|
||||
.retry_for_operator(&previous.observed_source_generation)
|
||||
.expect("exact retained generation should be retryable");
|
||||
previous
|
||||
.validate_successor(&retained)
|
||||
.expect("operator retry should be a valid successor");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(retained.revision, previous.revision + 1);
|
||||
assert_eq!(retained.attempt_count, previous.attempt_count);
|
||||
assert_eq!(retained.first_failure_at_unix_nanos, previous.first_failure_at_unix_nanos);
|
||||
assert_eq!(retained.last_failure_at_unix_nanos, previous.last_failure_at_unix_nanos);
|
||||
assert_eq!(retained.last_error_code, previous.last_error_code);
|
||||
assert_eq!(retained.consecutive_failure_count, 0);
|
||||
assert_eq!(retained.next_attempt_at_unix_nanos, None);
|
||||
assert!(retained.should_attempt_at(2_000_000_002));
|
||||
|
||||
if classification == IlmRecoveryClassification::OperatorRequired {
|
||||
retained
|
||||
.claim("node-b", Uuid::new_v4(), 2_000_000_002, 1)
|
||||
.expect("operator retry should authorize one new bounded attempt");
|
||||
retained
|
||||
.record_retryable_failure(2_000_000_003, IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("the bounded attempt failure should persist");
|
||||
assert_eq!(retained.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
assert_eq!(retained.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS) + 1);
|
||||
assert_eq!(retained.consecutive_failure_count, 1);
|
||||
}
|
||||
|
||||
let mut wrong_generation = previous;
|
||||
let mut changed_generation = wrong_generation.observed_source_generation.clone();
|
||||
changed_generation.source_etag = "changed".to_string();
|
||||
assert!(wrong_generation.retry_for_operator(&changed_generation).is_err());
|
||||
}
|
||||
|
||||
let mut exhausted = control();
|
||||
exhausted.classification = IlmRecoveryClassification::OperatorRequired;
|
||||
exhausted.attempt_count = u64::MAX;
|
||||
let generation = exhausted.observed_source_generation.clone();
|
||||
assert!(exhausted.retry_for_operator(&generation).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_control_view_redacts_source_and_owner_details() {
|
||||
let mut control = control();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,840 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::config_boundary;
|
||||
use super::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryProtocol, IlmRecoverySourceCopy, IlmRecoverySourceGeneration,
|
||||
MAX_ILM_RECOVERY_CONTROL_SIZE, ObservedIlmRecoveryControl, ObservedIlmRecoverySource, recovery_control_record_object_name,
|
||||
};
|
||||
use super::tier_delete_journal::{
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_source,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{ObjectOptions, WriteCompletion};
|
||||
use crate::services::notification_sys::{
|
||||
acquire_ilm_recovery_export_fleet_proof, ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_member_epochs_sha256,
|
||||
ilm_recovery_export_topology_generation,
|
||||
};
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, namespace::NamespaceLocking as _, object::HTTPPreconditions};
|
||||
use crate::store::ECStore;
|
||||
|
||||
pub const ILM_RECOVERY_EXPORT_SCHEMA: &str = "rustfs-ilm-recovery-export-v1";
|
||||
pub const ILM_RECOVERY_EXPORT_PREFIX: &str = "ilm/recovery-exports";
|
||||
pub const MAX_ILM_RECOVERY_EXPORT_SIZE: usize = 128 * 1024;
|
||||
const MAX_ILM_RECOVERY_EXPORTS: usize = 10_000;
|
||||
const MAX_ILM_RECOVERY_EXPORT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
const MAX_ACTOR_EXPORTS_PER_MINUTE: usize = 10;
|
||||
const MAX_CLUSTER_EXPORTS_PER_MINUTE: usize = 100;
|
||||
const EXPORT_RETENTION_NANOS: i64 = 90 * 24 * 60 * 60 * 1_000_000_000;
|
||||
const EXPORT_ADMISSION_LOCK: &str = "ilm/recovery-admission/export.lock";
|
||||
const MAX_LEGACY_TIER_DELETE_SOURCE_SIZE: usize = 64 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExportObservation {
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IlmRecoveryExport {
|
||||
pub export_id: String,
|
||||
pub control_id: String,
|
||||
pub protocol: IlmRecoveryProtocol,
|
||||
pub control_etag: String,
|
||||
pub control_revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub canonical_source_path: String,
|
||||
pub source_generation: IlmRecoverySourceGeneration,
|
||||
pub topology_generation: String,
|
||||
pub member_epochs_sha256: String,
|
||||
pub creator_sha256: String,
|
||||
pub created_at_unix_nanos: i64,
|
||||
pub retain_until_unix_nanos: i64,
|
||||
pub source_bytes_base64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedIlmRecoveryExport {
|
||||
schema: String,
|
||||
content_sha256: String,
|
||||
export: IlmRecoveryExport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct IlmRecoveryExportCreated {
|
||||
pub export_id: String,
|
||||
pub content_sha256: String,
|
||||
pub encoded: Vec<u8>,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExport {
|
||||
fn validate(&self) -> Result<()> {
|
||||
self.source_generation.validate().map_err(Error::other)?;
|
||||
validate_sha256(&self.export_id, "ILM recovery export ID is invalid")?;
|
||||
validate_sha256(&self.control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&self.topology_generation, "ILM recovery export topology generation is invalid")?;
|
||||
validate_sha256(&self.member_epochs_sha256, "ILM recovery export member epoch digest is invalid")?;
|
||||
validate_sha256(&self.creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
if self.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| self.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&self.source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source is not an exportable legacy journal"));
|
||||
}
|
||||
if self.control_etag.trim().is_empty() || self.control_revision == 0 {
|
||||
return Err(Error::other("ILM recovery export control generation is invalid"));
|
||||
}
|
||||
if self.canonical_source_path.is_empty()
|
||||
|| self.canonical_source_path.starts_with('/')
|
||||
|| self.canonical_source_path.ends_with('/')
|
||||
|| self.canonical_source_path.split('/').any(str::is_empty)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source path is invalid"));
|
||||
}
|
||||
if self.created_at_unix_nanos <= 0
|
||||
|| self.retain_until_unix_nanos < self.created_at_unix_nanos.saturating_add(EXPORT_RETENTION_NANOS)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export retention is invalid"));
|
||||
}
|
||||
let source = base64_simd::STANDARD
|
||||
.decode_to_vec(self.source_bytes_base64.as_bytes())
|
||||
.map_err(|_| Error::other("ILM recovery export source encoding is invalid"))?;
|
||||
validate_legacy_tier_delete_recovery_source(&self.canonical_source_path, &self.source_generation.source_schema, &source)?;
|
||||
let encoded_len = u64::try_from(source.len()).map_err(|_| Error::other("ILM recovery export source length overflow"))?;
|
||||
if source.is_empty()
|
||||
|| source.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE
|
||||
|| hex_sha256(&source, ToOwned::to_owned) != self.source_generation.content_sha256
|
||||
|| self.source_generation.copies.iter().any(|copy| {
|
||||
copy.canonical_path != self.canonical_source_path
|
||||
|| copy.etag != self.source_generation.source_etag
|
||||
|| copy.content_sha256 != self.source_generation.content_sha256
|
||||
|| copy.encoded_len != encoded_len
|
||||
})
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source bytes do not match the observed generation"));
|
||||
}
|
||||
if recovery_export_id(&self.control_id, &self.source_generation)? != self.export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match its source generation"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
let export_bytes = serde_json::to_vec(self).map_err(Error::other)?;
|
||||
let persisted = PersistedIlmRecoveryExport {
|
||||
schema: ILM_RECOVERY_EXPORT_SCHEMA.to_string(),
|
||||
content_sha256: hex_sha256(&export_bytes, ToOwned::to_owned),
|
||||
export: self.clone(),
|
||||
};
|
||||
let encoded = serde_json::to_vec(&persisted).map_err(Error::other)?;
|
||||
if encoded.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
pub fn decode(expected_export_id: &str, data: &[u8]) -> Result<Self> {
|
||||
validate_sha256(expected_export_id, "ILM recovery export ID is invalid")?;
|
||||
if data.len() > MAX_ILM_RECOVERY_EXPORT_SIZE {
|
||||
return Err(Error::other("encoded ILM recovery export exceeds maximum size"));
|
||||
}
|
||||
let persisted: PersistedIlmRecoveryExport = serde_json::from_slice(data).map_err(Error::other)?;
|
||||
if persisted.schema != ILM_RECOVERY_EXPORT_SCHEMA {
|
||||
return Err(Error::other("ILM recovery export schema is unsupported"));
|
||||
}
|
||||
validate_sha256(&persisted.content_sha256, "ILM recovery export checksum is invalid")?;
|
||||
let export_bytes = serde_json::to_vec(&persisted.export).map_err(Error::other)?;
|
||||
if hex_sha256(&export_bytes, ToOwned::to_owned) != persisted.content_sha256 {
|
||||
return Err(Error::other("ILM recovery export checksum mismatch"));
|
||||
}
|
||||
persisted.export.validate()?;
|
||||
if persisted.export.export_id != expected_export_id {
|
||||
return Err(Error::other("ILM recovery export ID does not match record key"));
|
||||
}
|
||||
Ok(persisted.export)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recovery_export_record_object_name(protocol: IlmRecoveryProtocol, export_id: &str) -> Result<String> {
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
Ok(format!(
|
||||
"{}/{}/{}/{}/{}.json",
|
||||
ILM_RECOVERY_EXPORT_PREFIX,
|
||||
protocol.as_str(),
|
||||
&export_id[..2],
|
||||
&export_id[2..4],
|
||||
export_id
|
||||
))
|
||||
}
|
||||
|
||||
pub fn recovery_export_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> {
|
||||
let suffix = object
|
||||
.strip_prefix(ILM_RECOVERY_EXPORT_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.ok_or_else(|| Error::other("ILM recovery export path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let protocol = match parts.next() {
|
||||
Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal,
|
||||
_ => return Err(Error::other("ILM recovery export protocol is invalid")),
|
||||
};
|
||||
let shard_a = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?;
|
||||
let export_id = parts
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.ok_or_else(|| Error::other("ILM recovery export suffix is invalid"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(Error::other("ILM recovery export path is not canonical"));
|
||||
}
|
||||
validate_sha256(export_id, "ILM recovery export ID is invalid")?;
|
||||
if shard_a != &export_id[..2] || shard_b != &export_id[2..4] {
|
||||
return Err(Error::other("ILM recovery export shard does not match export ID"));
|
||||
}
|
||||
Ok((protocol, export_id.to_string()))
|
||||
}
|
||||
|
||||
pub async fn inspect_recovery_export_observation(api: Arc<ECStore>, control_id: &str) -> Result<IlmRecoveryExportObservation> {
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
let observed_control = load_exportable_control(api.clone(), control_id).await?;
|
||||
let observed_source = observe_export_source(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(&proof).await
|
||||
{
|
||||
return Err(Error::other("ILM recovery export observation changed or is incomplete"));
|
||||
}
|
||||
Ok(IlmRecoveryExportObservation {
|
||||
control_id: control_id.to_string(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(&proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(&proof),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn create_recovery_export(
|
||||
api: Arc<ECStore>,
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
) -> Result<IlmRecoveryExportCreated> {
|
||||
validate_sha256(creator_sha256, "ILM recovery export creator digest is invalid")?;
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, EXPORT_ADMISSION_LOCK).await?;
|
||||
let admission_guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
|
||||
let proof = acquire_ilm_recovery_export_fleet_proof()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?;
|
||||
if ilm_recovery_export_topology_generation(&proof) != observation.topology_generation
|
||||
|| ilm_recovery_export_member_epochs_sha256(&proof) != observation.member_epochs_sha256
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.map_err(Error::other)?;
|
||||
let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let control_guard = control_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
let source_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &observation.canonical_source_path)
|
||||
.await?;
|
||||
let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let locks_current = || !admission_guard.is_lock_lost() && !control_guard.is_lock_lost() && !source_guard.is_lock_lost();
|
||||
let (current, current_source_bytes) = current_observation_under_proof_no_lock(api.clone(), observation, &proof).await?;
|
||||
if ¤t != observation || !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let current_source_base64 = base64_simd::STANDARD.encode_to_string(current_source_bytes);
|
||||
let candidate_export_id = recovery_export_id(¤t.control_id, ¤t.source_generation)?;
|
||||
let object = recovery_export_record_object_name(current.protocol, &candidate_export_id)?;
|
||||
match load_recovery_export_decoded(api.clone(), &candidate_export_id).await {
|
||||
Ok((existing, export)) if export_matches_observation(&export, observation) => {
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &existing.encoded)
|
||||
.await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
return Ok(existing.with_replayed());
|
||||
}
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let inventory = collect_export_inventory(api.clone()).await?;
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let created_at_unix_nanos = now_unix_nanos()?;
|
||||
let export = build_export_from_source(¤t, creator_sha256, created_at_unix_nanos, ¤t_source_base64)?;
|
||||
let encoded = export.encode()?;
|
||||
inventory.check(creator_sha256, encoded.len(), created_at_unix_nanos)?;
|
||||
|
||||
let mut write_options = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
write_options.add_namespace_lock_guard(&admission_guard);
|
||||
write_options.add_namespace_lock_guard(&control_guard);
|
||||
write_options.add_namespace_lock_guard(&source_guard);
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded.clone(), &write_options).await;
|
||||
let stored = match load_recovery_export(api.clone(), &export.export_id).await {
|
||||
Ok(stored) if stored.encoded == encoded => stored,
|
||||
Ok(_) => return Err(Error::PreconditionFailed),
|
||||
Err(read_err) => return Err(write_result.err().unwrap_or(read_err)),
|
||||
};
|
||||
if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
api.record_durable_ilm_decommission_progress(&object, &encoded).await?;
|
||||
if !locks_current() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
pub async fn load_recovery_export(api: Arc<ECStore>, export_id: &str) -> Result<IlmRecoveryExportCreated> {
|
||||
let (created, _) = load_recovery_export_decoded(api, export_id).await?;
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
async fn load_recovery_export_decoded(
|
||||
api: Arc<ECStore>,
|
||||
export_id: &str,
|
||||
) -> Result<(IlmRecoveryExportCreated, IlmRecoveryExport)> {
|
||||
let object = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, export_id)?;
|
||||
let encoded = config_boundary::read_config_limited_preserve_empty(api, &object, MAX_ILM_RECOVERY_EXPORT_SIZE).await?;
|
||||
let export = IlmRecoveryExport::decode(export_id, &encoded)?;
|
||||
let content_sha256 = hex_sha256(&encoded, ToOwned::to_owned);
|
||||
Ok((
|
||||
IlmRecoveryExportCreated {
|
||||
export_id: export.export_id.clone(),
|
||||
content_sha256,
|
||||
encoded,
|
||||
replayed: false,
|
||||
},
|
||||
export,
|
||||
))
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportCreated {
|
||||
fn with_replayed(mut self) -> Self {
|
||||
self.replayed = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_exportable_control(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(api, control_id, &ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_no_lock(api: Arc<ECStore>, control_id: &str) -> Result<ObservedIlmRecoveryControl> {
|
||||
load_exportable_control_with_options(
|
||||
api,
|
||||
control_id,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_exportable_control_with_options(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
options: &ObjectOptions,
|
||||
) -> Result<ObservedIlmRecoveryControl> {
|
||||
let object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, control_id).map_err(Error::other)?;
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_limited_preserve_empty_with_metadata(api, &object, options, MAX_ILM_RECOVERY_CONTROL_SIZE)
|
||||
.await?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery control is missing an ETag"))?;
|
||||
let control = IlmRecoveryControl::decode(control_id, &data).map_err(Error::other)?;
|
||||
if control.identity.protocol != IlmRecoveryProtocol::TierDeleteJournal
|
||||
|| control.classification != IlmRecoveryClassification::RetainedAmbiguous
|
||||
|| !is_legacy_export_schema(&control.observed_source_generation.source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery control is not exportable"));
|
||||
}
|
||||
Ok(ObservedIlmRecoveryControl { control, etag })
|
||||
}
|
||||
|
||||
async fn current_observation_under_proof_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
expected: &IlmRecoveryExportObservation,
|
||||
proof: &crate::services::notification_sys::IlmRecoveryExportFleetProofToken,
|
||||
) -> Result<(IlmRecoveryExportObservation, Vec<u8>)> {
|
||||
let observed_control = load_exportable_control_no_lock(api.clone(), &expected.control_id).await?;
|
||||
let observed_source = observe_export_source_no_lock(
|
||||
api,
|
||||
&observed_control.control.identity.canonical_source_path,
|
||||
&observed_control.control.observed_source_generation.source_schema,
|
||||
)
|
||||
.await?;
|
||||
let source_bytes = observed_source
|
||||
.canonical_data
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copies diverge"))?;
|
||||
if !observed_source.is_consistent()
|
||||
|| observed_source.generation != observed_control.control.observed_source_generation
|
||||
|| !ilm_recovery_export_fleet_proof_matches(proof).await
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok((
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: expected.control_id.clone(),
|
||||
protocol: observed_control.control.identity.protocol,
|
||||
control_etag: observed_control.etag,
|
||||
control_revision: observed_control.control.revision,
|
||||
classification: observed_control.control.classification,
|
||||
canonical_source_path: observed_control.control.identity.canonical_source_path,
|
||||
source_generation: observed_source.generation,
|
||||
topology_generation: ilm_recovery_export_topology_generation(proof),
|
||||
member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(proof),
|
||||
},
|
||||
source_bytes,
|
||||
))
|
||||
}
|
||||
|
||||
async fn observe_export_source(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
if canonical_path.is_empty()
|
||||
|| canonical_path.starts_with('/')
|
||||
|| canonical_path.ends_with('/')
|
||||
|| canonical_path.split('/').any(str::is_empty)
|
||||
|| !is_legacy_export_schema(source_schema)
|
||||
{
|
||||
return Err(Error::other("ILM recovery export source identity is invalid"));
|
||||
}
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, canonical_path).await?;
|
||||
let _guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
observe_export_source_no_lock(api, canonical_path, source_schema).await
|
||||
}
|
||||
|
||||
async fn observe_export_source_no_lock(
|
||||
api: Arc<ECStore>,
|
||||
canonical_path: &str,
|
||||
source_schema: &str,
|
||||
) -> Result<ObservedIlmRecoverySource> {
|
||||
let mut copies = Vec::new();
|
||||
let mut canonical: Option<(String, String, Vec<u8>)> = None;
|
||||
let mut consistent = true;
|
||||
for set in api.all_set_disks() {
|
||||
let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index);
|
||||
let result = config_boundary::read_config_limited_preserve_empty_with_metadata(
|
||||
set,
|
||||
canonical_path,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
MAX_LEGACY_TIER_DELETE_SOURCE_SIZE,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok((data, metadata)) => {
|
||||
if data.is_empty() || data.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE {
|
||||
return Err(Error::other("ILM recovery export source exceeds its protocol size limit"));
|
||||
}
|
||||
validate_legacy_tier_delete_recovery_source(canonical_path, source_schema, &data)?;
|
||||
let etag = metadata
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("ILM recovery export source copy is missing an ETag"))?;
|
||||
let content_sha256 = hex_sha256(&data, ToOwned::to_owned);
|
||||
let encoded_len =
|
||||
u64::try_from(data.len()).map_err(|_| Error::other("ILM recovery export source length does not fit u64"))?;
|
||||
copies.push(IlmRecoverySourceCopy {
|
||||
authority,
|
||||
canonical_path: canonical_path.to_string(),
|
||||
etag: etag.clone(),
|
||||
encoded_len,
|
||||
content_sha256: content_sha256.clone(),
|
||||
});
|
||||
match canonical.as_ref() {
|
||||
Some((first_etag, first_digest, first_data)) => {
|
||||
consistent &= first_etag == &etag && first_digest == &content_sha256 && first_data == &data;
|
||||
}
|
||||
None => canonical = Some((etag, content_sha256, data)),
|
||||
}
|
||||
}
|
||||
Err(err) if export_source_is_missing(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let Some((source_etag, content_sha256, source_bytes)) = canonical else {
|
||||
return Err(Error::ConfigNotFound);
|
||||
};
|
||||
let generation =
|
||||
IlmRecoverySourceGeneration::new(source_schema, source_etag, content_sha256, copies).map_err(Error::other)?;
|
||||
Ok(ObservedIlmRecoverySource {
|
||||
generation,
|
||||
canonical_data: consistent.then_some(source_bytes),
|
||||
})
|
||||
}
|
||||
|
||||
fn export_source_is_missing(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _)
|
||||
)
|
||||
}
|
||||
|
||||
fn build_export_from_source(
|
||||
observation: &IlmRecoveryExportObservation,
|
||||
creator_sha256: &str,
|
||||
created_at_unix_nanos: i64,
|
||||
source_bytes_base64: &str,
|
||||
) -> Result<IlmRecoveryExport> {
|
||||
let retain_until_unix_nanos = created_at_unix_nanos
|
||||
.checked_add(EXPORT_RETENTION_NANOS)
|
||||
.ok_or_else(|| Error::other("ILM recovery export retention timestamp overflow"))?;
|
||||
let export = IlmRecoveryExport {
|
||||
export_id: recovery_export_id(&observation.control_id, &observation.source_generation)?,
|
||||
control_id: observation.control_id.clone(),
|
||||
protocol: observation.protocol,
|
||||
control_etag: observation.control_etag.clone(),
|
||||
control_revision: observation.control_revision,
|
||||
classification: observation.classification,
|
||||
canonical_source_path: observation.canonical_source_path.clone(),
|
||||
source_generation: observation.source_generation.clone(),
|
||||
topology_generation: observation.topology_generation.clone(),
|
||||
member_epochs_sha256: observation.member_epochs_sha256.clone(),
|
||||
creator_sha256: creator_sha256.to_string(),
|
||||
created_at_unix_nanos,
|
||||
retain_until_unix_nanos,
|
||||
source_bytes_base64: source_bytes_base64.to_string(),
|
||||
};
|
||||
export.validate()?;
|
||||
Ok(export)
|
||||
}
|
||||
|
||||
pub(crate) fn recovery_export_id(control_id: &str, generation: &IlmRecoverySourceGeneration) -> Result<String> {
|
||||
validate_sha256(control_id, "ILM recovery export control ID is invalid")?;
|
||||
validate_sha256(&generation.content_sha256, "ILM recovery export source checksum is invalid")?;
|
||||
validate_sha256(&generation.copy_set_sha256, "ILM recovery export copy-set checksum is invalid")?;
|
||||
let mut data = Vec::new();
|
||||
for part in [control_id, &generation.content_sha256, &generation.copy_set_sha256] {
|
||||
data.extend_from_slice(&(part.len() as u64).to_be_bytes());
|
||||
data.extend_from_slice(part.as_bytes());
|
||||
}
|
||||
Ok(hex_sha256(&data, ToOwned::to_owned))
|
||||
}
|
||||
|
||||
fn export_matches_observation(export: &IlmRecoveryExport, observation: &IlmRecoveryExportObservation) -> bool {
|
||||
export.control_id == observation.control_id
|
||||
&& export.protocol == observation.protocol
|
||||
&& export.classification == observation.classification
|
||||
&& export.canonical_source_path == observation.canonical_source_path
|
||||
&& export.source_generation == observation.source_generation
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct IlmRecoveryExportInventory {
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
creations: Vec<(i64, String)>,
|
||||
}
|
||||
|
||||
impl IlmRecoveryExportInventory {
|
||||
fn check(&self, creator_sha256: &str, candidate_len: usize, now: i64) -> Result<()> {
|
||||
let recent_after = now.saturating_sub(60 * 1_000_000_000);
|
||||
let cluster_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, _)| *created_at > recent_after)
|
||||
.count();
|
||||
let actor_recent = self
|
||||
.creations
|
||||
.iter()
|
||||
.filter(|(created_at, creator)| *created_at > recent_after && creator == creator_sha256)
|
||||
.count();
|
||||
check_export_admission(self.count, self.bytes, actor_recent, cluster_recent, candidate_len)
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_export_inventory(api: Arc<ECStore>) -> Result<IlmRecoveryExportInventory> {
|
||||
let mut marker = None;
|
||||
let mut seen_markers = HashSet::new();
|
||||
let mut inventory = IlmRecoveryExportInventory::default();
|
||||
loop {
|
||||
let page = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("{ILM_RECOVERY_EXPORT_PREFIX}/"),
|
||||
marker.clone(),
|
||||
None,
|
||||
1_000,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
for object in page.objects {
|
||||
let (_, export_id) = recovery_export_id_from_record_object_name(&object.name)?;
|
||||
let (stored, export) = load_recovery_export_decoded(api.clone(), &export_id).await?;
|
||||
inventory.count = inventory
|
||||
.count
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| Error::other("ILM recovery export count overflow"))?;
|
||||
inventory.bytes = inventory
|
||||
.bytes
|
||||
.checked_add(u64::try_from(stored.encoded.len()).map_err(|_| Error::other("ILM recovery export size overflow"))?)
|
||||
.ok_or_else(|| Error::other("ILM recovery export byte total overflow"))?;
|
||||
inventory
|
||||
.creations
|
||||
.push((export.created_at_unix_nanos, export.creator_sha256));
|
||||
}
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
let next = page
|
||||
.next_continuation_token
|
||||
.ok_or_else(|| Error::other("ILM recovery export inventory omitted its continuation marker"))?;
|
||||
marker = Some(record_export_inventory_marker(&mut seen_markers, next)?);
|
||||
}
|
||||
Ok(inventory)
|
||||
}
|
||||
|
||||
fn record_export_inventory_marker(seen_markers: &mut HashSet<String>, next: String) -> Result<String> {
|
||||
if !seen_markers.insert(next.clone()) {
|
||||
return Err(Error::other("ILM recovery export inventory repeated its continuation marker"));
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
fn check_export_admission(
|
||||
count: usize,
|
||||
bytes: u64,
|
||||
actor_recent: usize,
|
||||
cluster_recent: usize,
|
||||
candidate_len: usize,
|
||||
) -> Result<()> {
|
||||
let candidate_len = u64::try_from(candidate_len).map_err(|_| Error::other("ILM recovery export size does not fit u64"))?;
|
||||
if count >= MAX_ILM_RECOVERY_EXPORTS
|
||||
|| bytes
|
||||
.checked_add(candidate_len)
|
||||
.is_none_or(|total| total > MAX_ILM_RECOVERY_EXPORT_BYTES)
|
||||
|| actor_recent >= MAX_ACTOR_EXPORTS_PER_MINUTE
|
||||
|| cluster_recent >= MAX_CLUSTER_EXPORTS_PER_MINUTE
|
||||
{
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_legacy_export_schema(schema: &str) -> bool {
|
||||
matches!(schema, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA | TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA)
|
||||
}
|
||||
|
||||
fn validate_sha256(value: &str, message: &'static str) -> Result<()> {
|
||||
if !is_sha256_checksum(value)
|
||||
|| value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(Error::other(message));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn now_unix_nanos() -> Result<i64> {
|
||||
i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.map_err(|_| Error::other("ILM recovery export timestamp does not fit i64"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy;
|
||||
|
||||
const PINNED_V1_EXPORT: &[u8] = br#"{"schema":"rustfs-ilm-recovery-export-v1","content_sha256":"3dfb3ec3892256e909de1211c1a963ca7008963ff32b3a869f7161a7b9b44028","export":{"export_id":"2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105","control_id":"0fcd568a5cb9bdb4677b69354b11ee415af8f784519cff3da49a26f84eaee7f2","protocol":"tier_delete_journal","control_etag":"control-etag","control_revision":1,"classification":"retained_ambiguous","canonical_source_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","source_generation":{"source_schema":"rustfs-tier-delete-journal-v1","source_etag":"etag-a","content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd","copy_set_sha256":"5a7406115b6c3923ffe79dcd1f43ccae7beed786e557163f019dd10ec409a653","copies":[{"authority":"pool-0/set-0","canonical_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","etag":"etag-a","encoded_len":81,"content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd"}]},"topology_generation":"e6e2b826e31fca5c36125c48f130dcb6f961e698ff8a8776a1f290cf0892e8e6","member_epochs_sha256":"612dd8a861161819a4ad8f6f3e2a0567602877c043a2353ca933a13c78dc0ed4","creator_sha256":"50c9c4aeb40b5b206b6d98f516f8b8c0efd29ce2e56a76b345fb9240c225a1b7","created_at_unix_nanos":1000000000,"retain_until_unix_nanos":7776001000000000,"source_bytes_base64":"eyJ2ZXJzaW9uIjoxLCJvYmpfbmFtZSI6ImxlZ2FjeS9yZW1vdGUiLCJ2ZXJzaW9uX2lkIjoib3BhcXVlIiwidGllcl9uYW1lIjoiV0FSTSJ9"}}"#;
|
||||
|
||||
fn legacy_source() -> Vec<u8> {
|
||||
br#"{"version":1,"obj_name":"legacy/remote","version_id":"opaque","tier_name":"WARM"}"#.to_vec()
|
||||
}
|
||||
|
||||
fn observation() -> IlmRecoveryExportObservation {
|
||||
let source = legacy_source();
|
||||
let source_path = super::super::tier_delete_journal::tier_delete_journal_object_name(
|
||||
&super::super::tier_delete_journal::decode_tier_delete_journal_entry(&source).expect("legacy fixture should decode"),
|
||||
);
|
||||
let source_sha256 = hex_sha256(&source, ToOwned::to_owned);
|
||||
let generation = IlmRecoverySourceGeneration::new(
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA,
|
||||
"etag-a",
|
||||
source_sha256.clone(),
|
||||
vec![IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.clone(),
|
||||
etag: "etag-a".to_string(),
|
||||
encoded_len: source.len() as u64,
|
||||
content_sha256: source_sha256,
|
||||
}],
|
||||
)
|
||||
.expect("generation should be valid");
|
||||
IlmRecoveryExportObservation {
|
||||
control_id: hex_sha256(b"control", ToOwned::to_owned),
|
||||
protocol: IlmRecoveryProtocol::TierDeleteJournal,
|
||||
control_etag: "control-etag".to_string(),
|
||||
control_revision: 1,
|
||||
classification: IlmRecoveryClassification::RetainedAmbiguous,
|
||||
canonical_source_path: source_path,
|
||||
source_generation: generation,
|
||||
topology_generation: hex_sha256(b"topology", ToOwned::to_owned),
|
||||
member_epochs_sha256: hex_sha256(b"epochs", ToOwned::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_round_trip_is_strict_and_deterministic() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.expect("export should be valid");
|
||||
assert_eq!(
|
||||
export.export_id,
|
||||
recovery_export_id(&observed.control_id, &observed.source_generation).unwrap()
|
||||
);
|
||||
let encoded = export.encode().expect("export should encode");
|
||||
assert_eq!(encoded, PINNED_V1_EXPORT, "v1 export wire format must remain pinned");
|
||||
assert_eq!(IlmRecoveryExport::decode(&export.export_id, &encoded).unwrap(), export);
|
||||
assert_eq!(
|
||||
IlmRecoveryExport::decode("2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105", PINNED_V1_EXPORT)
|
||||
.unwrap(),
|
||||
export,
|
||||
);
|
||||
|
||||
let path = recovery_export_record_object_name(export.protocol, &export.export_id).unwrap();
|
||||
let durable = super::super::durable_namespace::validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("export should be registered as a durable ILM record");
|
||||
assert_eq!(durable.namespace, "recovery-export");
|
||||
assert_eq!(durable.id_kind, "export_id");
|
||||
assert_eq!(durable.id, export.export_id);
|
||||
|
||||
let mut wrong_source = export.clone();
|
||||
wrong_source.source_bytes_base64 = base64_simd::STANDARD.encode_to_string(b"changed");
|
||||
assert!(wrong_source.encode().is_err());
|
||||
|
||||
let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
|
||||
persisted["unknown"] = serde_json::json!(true);
|
||||
assert!(IlmRecoveryExport::decode(&export.export_id, &serde_json::to_vec(&persisted).unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_inventory_rejects_non_adjacent_continuation_cycles() {
|
||||
let mut seen = HashSet::new();
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "a".to_string()).unwrap(), "a");
|
||||
assert_eq!(record_export_inventory_marker(&mut seen, "b".to_string()).unwrap(), "b");
|
||||
record_export_inventory_marker(&mut seen, "a".to_string())
|
||||
.expect_err("a non-adjacent continuation marker cycle must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_export_path_rejects_noncanonical_shards() {
|
||||
let id = hex_sha256(b"export", ToOwned::to_owned);
|
||||
let path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &id).unwrap();
|
||||
assert_eq!(recovery_export_id_from_record_object_name(&path).unwrap().1, id);
|
||||
let wrong_shard = path.replacen(&format!("/{}/", &id[..2]), "/zz/", 1);
|
||||
assert!(recovery_export_id_from_record_object_name(&wrong_shard).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_replay_survives_fleet_rotation_but_not_source_change() {
|
||||
let observed = observation();
|
||||
let creator = hex_sha256(b"actor", ToOwned::to_owned);
|
||||
let export = build_export_from_source(
|
||||
&observed,
|
||||
&creator,
|
||||
1_000_000_000,
|
||||
&base64_simd::STANDARD.encode_to_string(legacy_source()),
|
||||
)
|
||||
.unwrap();
|
||||
let mut rotated = observed;
|
||||
rotated.control_etag = "new-control-etag".to_string();
|
||||
rotated.control_revision += 1;
|
||||
rotated.topology_generation = hex_sha256(b"new-topology", ToOwned::to_owned);
|
||||
rotated.member_epochs_sha256 = hex_sha256(b"new-members", ToOwned::to_owned);
|
||||
assert!(export_matches_observation(&export, &rotated));
|
||||
|
||||
rotated.source_generation.content_sha256 = hex_sha256(b"changed", ToOwned::to_owned);
|
||||
assert!(!export_matches_observation(&export, &rotated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_admission_enforces_exact_count_byte_and_rate_boundaries() {
|
||||
assert!(check_export_admission(9_999, MAX_ILM_RECOVERY_EXPORT_BYTES - 1, 9, 99, 1).is_ok());
|
||||
assert!(check_export_admission(10_000, 0, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, MAX_ILM_RECOVERY_EXPORT_BYTES, 0, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 10, 0, 1).is_err());
|
||||
assert!(check_export_admission(0, 0, 0, 100, 1).is_err());
|
||||
}
|
||||
}
|
||||
@@ -82,8 +82,8 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
|
||||
const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
pub(crate) const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
|
||||
const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
|
||||
const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
|
||||
const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
|
||||
@@ -884,23 +884,6 @@ struct PersistedTierDeleteJournalEntry {
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn validate_legacy_recovery_shape(&self) -> Result<()> {
|
||||
let has_later_version_fields = self.version_id_exact.is_some()
|
||||
|| self.version_state.is_some()
|
||||
|| self.state.is_some()
|
||||
|| self.source.is_some()
|
||||
|| self.dispatch.is_some();
|
||||
match self.version {
|
||||
1 if self.backend_identity.is_none() && !has_later_version_fields => Ok(()),
|
||||
TIER_DELETE_JOURNAL_VERSION if self.backend_identity.is_some() && !has_later_version_fields => Ok(()),
|
||||
1 => Err(Error::other("tier delete journal v1 entry contains fields from a later version")),
|
||||
TIER_DELETE_JOURNAL_VERSION => Err(Error::other(
|
||||
"tier delete journal v2 entry is missing its identity or contains fields from a later version",
|
||||
)),
|
||||
_ => Err(Error::other("tier delete journal is not an exportable legacy version")),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
@@ -5548,12 +5531,6 @@ fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&s
|
||||
.then_some(identity)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_path(object_name: &str) -> Result<()> {
|
||||
canonical_legacy_tier_delete_journal_identity(object_name)
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| Error::other("legacy tier delete journal path is not canonical"))
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
|
||||
match entry.persisted_version {
|
||||
1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
|
||||
@@ -5562,21 +5539,6 @@ fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static st
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_legacy_tier_delete_recovery_source(object_name: &str, source_schema: &str, data: &[u8]) -> Result<()> {
|
||||
validate_legacy_tier_delete_recovery_path(object_name)?;
|
||||
let persisted: PersistedTierDeleteJournalEntry =
|
||||
serde_json::from_slice(data).map_err(|err| Error::other_with_context("decode tier delete journal failed", err))?;
|
||||
persisted.validate_legacy_recovery_shape()?;
|
||||
let entry = persisted.into_jentry()?;
|
||||
let Some((decoded_schema, _)) = legacy_tier_delete_recovery_descriptor(&entry) else {
|
||||
return Err(Error::other("tier delete journal is not an exportable legacy version"));
|
||||
};
|
||||
if decoded_schema != source_schema || tier_delete_journal_object_name(&entry) != object_name {
|
||||
return Err(Error::other("legacy tier delete journal identity does not match its recovery source"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_tier_delete_control_matches(
|
||||
control: &IlmRecoveryControl,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
@@ -6178,18 +6140,17 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
PersistedTierDeleteJournalEntry, TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE,
|
||||
TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX,
|
||||
TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION,
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TIER_DELETE_JOURNAL_VERSION, TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState,
|
||||
TierDeleteDispatchParent, TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION,
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX,
|
||||
TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent,
|
||||
TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest,
|
||||
encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity,
|
||||
tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name,
|
||||
tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name,
|
||||
tier_delete_source_matches_dispatch_scope, validate_legacy_tier_delete_recovery_source,
|
||||
tier_delete_source_matches_dispatch_scope,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -6648,72 +6609,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_recovery_export_rejects_fields_from_later_journal_versions() {
|
||||
let later = bound_v6_journal_entry(TierDeleteJournalState::Prepared);
|
||||
let v1 = PersistedTierDeleteJournalEntry {
|
||||
version: 1,
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "opaque".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: None,
|
||||
version_id_exact: None,
|
||||
version_state: None,
|
||||
state: None,
|
||||
source: None,
|
||||
dispatch: None,
|
||||
};
|
||||
let mut v2 = v1.clone();
|
||||
v2.version = TIER_DELETE_JOURNAL_VERSION;
|
||||
v2.backend_identity = Some([7; 32]);
|
||||
|
||||
let assert_rejected = |persisted: PersistedTierDeleteJournalEntry, schema: &str| {
|
||||
let normalized = persisted
|
||||
.clone()
|
||||
.into_jentry()
|
||||
.expect("the generic compatibility decoder should demonstrate the discarded field");
|
||||
let object_name = tier_delete_journal_object_name(&normalized);
|
||||
let encoded = serde_json::to_vec(&persisted).expect("mixed-version journal fixture should encode");
|
||||
let err = validate_legacy_tier_delete_recovery_source(&object_name, schema, &encoded)
|
||||
.expect_err("legacy recovery export must reject fields from later versions");
|
||||
assert!(err.to_string().contains("later version"));
|
||||
};
|
||||
|
||||
let mut invalid_v1 = Vec::new();
|
||||
let mut with_backend = v1.clone();
|
||||
with_backend.backend_identity = Some([7; 32]);
|
||||
invalid_v1.push(with_backend);
|
||||
for persisted in [&v1, &v2] {
|
||||
let schema = if persisted.version == 1 {
|
||||
TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA
|
||||
} else {
|
||||
TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA
|
||||
};
|
||||
let mut invalid = Vec::new();
|
||||
let mut with_exact = persisted.clone();
|
||||
with_exact.version_id_exact = Some(false);
|
||||
invalid.push(with_exact);
|
||||
let mut with_version_state = persisted.clone();
|
||||
with_version_state.version_state = Some(rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
invalid.push(with_version_state);
|
||||
let mut with_state = persisted.clone();
|
||||
with_state.state = Some(TierDeleteJournalState::Committed);
|
||||
invalid.push(with_state);
|
||||
let mut with_source = persisted.clone();
|
||||
with_source.source = later.source.clone();
|
||||
invalid.push(with_source);
|
||||
let mut with_dispatch = persisted.clone();
|
||||
with_dispatch.dispatch = later.dispatch.clone();
|
||||
invalid.push(with_dispatch);
|
||||
for record in invalid {
|
||||
assert_rejected(record, schema);
|
||||
}
|
||||
}
|
||||
for record in invalid_v1 {
|
||||
assert_rejected(record, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::bucket::lifecycle::tier_sweeper::{
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
@@ -273,14 +273,6 @@ pub struct TransitionTransactionInit {
|
||||
|
||||
impl TransitionTransaction {
|
||||
pub fn new(init: TransitionTransactionInit) -> Result<Self> {
|
||||
Self::new_with_initial_state(init, TransitionTransactionState::UploadStarted)
|
||||
}
|
||||
|
||||
pub(crate) fn new_compact(init: TransitionTransactionInit) -> Result<Self> {
|
||||
Self::new_with_initial_state(init, TransitionTransactionState::UploadOutcomeUnknown)
|
||||
}
|
||||
|
||||
fn new_with_initial_state(init: TransitionTransactionInit, state: TransitionTransactionState) -> Result<Self> {
|
||||
let remote_object =
|
||||
canonical_transition_remote_object(init.deployment_id, &init.source.bucket, init.transaction_id, init.write_id)?;
|
||||
let transaction = Self {
|
||||
@@ -294,7 +286,7 @@ impl TransitionTransaction {
|
||||
backend_fingerprint: init.backend_fingerprint,
|
||||
remote_object,
|
||||
remote_version: TransitionRemoteVersion::unknown(),
|
||||
state,
|
||||
state: TransitionTransactionState::UploadStarted,
|
||||
not_after_unix_nanos: init.not_after_unix_nanos,
|
||||
};
|
||||
transaction.validate()?;
|
||||
@@ -364,7 +356,7 @@ impl TransitionTransaction {
|
||||
remote_version: Option<TransitionRemoteVersion>,
|
||||
) -> Result<TransitionTransactionFence> {
|
||||
self.check_fence(fence)?;
|
||||
if !state_change_allowed_at(self.state, next, self.revision) {
|
||||
if !state_change_allowed(self.state, next) {
|
||||
return Err(TransitionTransactionError::InvalidStateChange {
|
||||
from: self.state,
|
||||
to: next,
|
||||
@@ -396,14 +388,6 @@ impl TransitionTransaction {
|
||||
}
|
||||
self.remote_version = TransitionRemoteVersion::unknown();
|
||||
}
|
||||
TransitionTransactionState::LocalCommitStarted if self.state == TransitionTransactionState::UploadOutcomeUnknown => {
|
||||
let remote_version =
|
||||
remote_version.ok_or(TransitionTransactionError::Corrupt("compact local commit requires remote version"))?;
|
||||
if remote_version.is_unknown() {
|
||||
return Err(TransitionTransactionError::Corrupt("compact local commit requires known remote version"));
|
||||
}
|
||||
self.remote_version = remote_version;
|
||||
}
|
||||
TransitionTransactionState::LocalCommitStarted | TransitionTransactionState::Committed => {
|
||||
if let Some(remote_version) = remote_version
|
||||
&& remote_version != self.remote_version
|
||||
@@ -656,7 +640,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
|
||||
) -> EcstoreResult<()> {
|
||||
let object = transition_transaction_record_object_name(next.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let revision_is_next = expected.revision.checked_add(1) == Some(next.revision);
|
||||
let state_is_next = state_change_allowed_at(expected.state, next.state, expected.revision)
|
||||
let state_is_next = state_change_allowed(expected.state, next.state)
|
||||
|| matches!(
|
||||
(expected.state, next.state),
|
||||
(
|
||||
@@ -970,10 +954,6 @@ pub enum TransitionOperatorError {
|
||||
expected: String,
|
||||
actual: TransitionOperatorProbe,
|
||||
},
|
||||
#[error("transition recovery control is stale")]
|
||||
StaleRecoveryControl,
|
||||
#[error("transition recovery control is not eligible for operator retry")]
|
||||
RetryNotAllowed,
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
@@ -982,179 +962,6 @@ pub enum TransitionOperatorError {
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionRecoveryRetryStatus {
|
||||
pub control_id: String,
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub control_revision: u64,
|
||||
pub attempt_count: u64,
|
||||
pub consecutive_failure_count: u32,
|
||||
pub last_error_code: IlmRecoveryErrorCode,
|
||||
pub source_generation_sha256: String,
|
||||
pub copy_set_sha256: String,
|
||||
pub retry_ready: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub retry_not_ready_reason: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionRecoveryRetryResult {
|
||||
pub control_id: String,
|
||||
pub transaction_id: Uuid,
|
||||
pub previous_revision: u64,
|
||||
pub revision: u64,
|
||||
pub classification: IlmRecoveryClassification,
|
||||
pub attempt_count: u64,
|
||||
pub source_generation_sha256: String,
|
||||
}
|
||||
|
||||
struct TransitionRecoveryRetryContext {
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
transaction: TransitionTransaction,
|
||||
source_generation_sha256: String,
|
||||
}
|
||||
|
||||
fn transition_recovery_retry_readiness(control: &IlmRecoveryControl) -> (bool, Option<&'static str>) {
|
||||
if control.owner.is_some() {
|
||||
return (false, Some("attempt_owned"));
|
||||
}
|
||||
match control.classification {
|
||||
IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired => (true, None),
|
||||
IlmRecoveryClassification::Retrying => (false, Some("already_retrying")),
|
||||
IlmRecoveryClassification::Corrupt => (false, Some("source_corrupt")),
|
||||
IlmRecoveryClassification::Abandoned => (false, Some("source_abandoned")),
|
||||
IlmRecoveryClassification::Terminal => (false, Some("source_terminal")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_transition_recovery_retry_context(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryContext> {
|
||||
let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(observed) => observed,
|
||||
Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
let transaction_id = Uuid::parse_str(&observed.control.identity.stable_operation_identity)
|
||||
.ok()
|
||||
.filter(|transaction_id| !transaction_id.is_nil())
|
||||
.ok_or(TransitionOperatorError::StaleRecoveryControl)?;
|
||||
let canonical_path = transition_transaction_record_object_name(transaction_id)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if observed.control.identity.canonical_source_path != canonical_path
|
||||
|| observed.control.identity.record_class != "transition_transaction_v1"
|
||||
{
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
|
||||
Ok(transaction) => transaction,
|
||||
Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
let source = observe_recovery_source(api, &canonical_path, TRANSITION_TRANSACTION_SCHEMA)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source.generation == observed.control.observed_source_generation
|
||||
&& source
|
||||
.canonical_data
|
||||
.as_deref()
|
||||
.is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
|
||||
if !exact_source {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let generation = serde_json::to_vec(&observed.control.observed_source_generation)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
Ok(TransitionRecoveryRetryContext {
|
||||
observed,
|
||||
transaction,
|
||||
source_generation_sha256: hex_sha256(&generation, ToOwned::to_owned),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_recovery_retry_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryStatus> {
|
||||
let context = load_transition_recovery_retry_context(api, control_id).await?;
|
||||
let (retry_ready, retry_not_ready_reason) = transition_recovery_retry_readiness(&context.observed.control);
|
||||
Ok(TransitionRecoveryRetryStatus {
|
||||
control_id: control_id.to_string(),
|
||||
transaction_id: context.transaction.transaction_id,
|
||||
state: context.transaction.state,
|
||||
classification: context.observed.control.classification,
|
||||
control_revision: context.observed.control.revision,
|
||||
attempt_count: context.observed.control.attempt_count,
|
||||
consecutive_failure_count: context.observed.control.consecutive_failure_count,
|
||||
last_error_code: context.observed.control.last_error_code,
|
||||
source_generation_sha256: context.source_generation_sha256,
|
||||
copy_set_sha256: context.observed.control.observed_source_generation.copy_set_sha256.clone(),
|
||||
retry_ready,
|
||||
retry_not_ready_reason,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn retry_transition_recovery_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
control_id: &str,
|
||||
expected_control_revision: u64,
|
||||
expected_source_generation_sha256: &str,
|
||||
) -> TransitionOperatorResult<TransitionRecoveryRetryResult> {
|
||||
let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, control_id)
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
let retry_lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_object}.recovery-lock"))
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
let retry_guard = retry_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
let context = load_transition_recovery_retry_context(api.clone(), control_id).await?;
|
||||
let (retry_ready, _) = transition_recovery_retry_readiness(&context.observed.control);
|
||||
if !retry_ready {
|
||||
return Err(TransitionOperatorError::RetryNotAllowed);
|
||||
}
|
||||
if retry_guard.is_lock_lost()
|
||||
|| expected_control_revision == 0
|
||||
|| context.observed.control.revision != expected_control_revision
|
||||
|| context.source_generation_sha256 != expected_source_generation_sha256
|
||||
{
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
let previous_revision = context.observed.control.revision;
|
||||
let mut next = context.observed.control.clone();
|
||||
next.retry_for_operator(&context.observed.control.observed_source_generation)
|
||||
.map_err(|_| TransitionOperatorError::RetryNotAllowed)?;
|
||||
if retry_guard.is_lock_lost() {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
save_recovery_control_if_current(api.clone(), &context.observed, &next)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
Error::PreconditionFailed => TransitionOperatorError::StaleRecoveryControl,
|
||||
err => TransitionOperatorError::Store(err),
|
||||
})?;
|
||||
let persisted = load_recovery_control(api, IlmRecoveryProtocol::TransitionTransaction, control_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)?;
|
||||
if retry_guard.is_lock_lost() || persisted.control != next {
|
||||
return Err(TransitionOperatorError::StaleRecoveryControl);
|
||||
}
|
||||
Ok(TransitionRecoveryRetryResult {
|
||||
control_id: control_id.to_string(),
|
||||
transaction_id: context.transaction.transaction_id,
|
||||
previous_revision,
|
||||
revision: persisted.control.revision,
|
||||
classification: persisted.control.classification,
|
||||
attempt_count: persisted.control.attempt_count,
|
||||
source_generation_sha256: context.source_generation_sha256,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
@@ -1899,27 +1706,12 @@ async fn local_commit_matches_transaction(api: Arc<ECStore>, transaction: &Trans
|
||||
.get_object_info(&transaction.source.bucket, &transaction.source.object, &opts)
|
||||
.await?;
|
||||
let transitioned = &object.transitioned_object;
|
||||
Ok(local_object_matches_transition_source(&object, &transaction.source)
|
||||
&& transitioned.status == TRANSITION_COMPLETE
|
||||
Ok(transitioned.status == TRANSITION_COMPLETE
|
||||
&& transitioned.name == transaction.remote_object
|
||||
&& transitioned.tier == transaction.tier_name
|
||||
&& transitioned.version_id == transaction.remote_version.tier_delete_version_id().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn local_object_matches_transition_source(object: &ObjectInfo, source: &TransitionSourceIdentity) -> bool {
|
||||
let observed_version_id = object.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let observed_mod_time = object
|
||||
.mod_time
|
||||
.and_then(|mod_time| i64::try_from(mod_time.unix_timestamp_nanos()).ok());
|
||||
object.bucket == source.bucket
|
||||
&& object.name == source.object
|
||||
&& observed_version_id == source.version_id
|
||||
&& object.data_dir == Some(source.data_dir)
|
||||
&& observed_mod_time == Some(source.mod_time_unix_nanos)
|
||||
&& object.size == source.size
|
||||
&& object.etag.as_deref() == Some(source.etag.as_str())
|
||||
}
|
||||
|
||||
fn transition_source_lookup_options(transaction: &TransitionTransaction) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
version_id: match transaction.source.version_mode {
|
||||
@@ -2162,7 +1954,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn state_change_allowed_at(from: TransitionTransactionState, to: TransitionTransactionState, revision: u64) -> bool {
|
||||
fn state_change_allowed(from: TransitionTransactionState, to: TransitionTransactionState) -> bool {
|
||||
matches!(
|
||||
(from, to),
|
||||
(TransitionTransactionState::UploadStarted, TransitionTransactionState::Uploaded)
|
||||
@@ -2174,9 +1966,7 @@ fn state_change_allowed_at(from: TransitionTransactionState, to: TransitionTrans
|
||||
| (TransitionTransactionState::UploadOutcomeUnknown, TransitionTransactionState::Uploaded)
|
||||
| (TransitionTransactionState::Uploaded, TransitionTransactionState::LocalCommitStarted)
|
||||
| (TransitionTransactionState::LocalCommitStarted, TransitionTransactionState::Committed)
|
||||
) || (revision == 1
|
||||
&& from == TransitionTransactionState::UploadOutcomeUnknown
|
||||
&& to == TransitionTransactionState::LocalCommitStarted)
|
||||
)
|
||||
}
|
||||
|
||||
fn state_requires_known_remote_version(state: TransitionTransactionState) -> bool {
|
||||
@@ -2393,41 +2183,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_commit_proof_requires_the_complete_source_identity() {
|
||||
let source = source_identity(TransitionSourceVersionMode::Versioned);
|
||||
let exact = ObjectInfo {
|
||||
bucket: source.bucket.clone(),
|
||||
name: source.object.clone(),
|
||||
version_id: source.version_id,
|
||||
data_dir: Some(source.data_dir),
|
||||
mod_time: Some(
|
||||
time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(source.mod_time_unix_nanos))
|
||||
.expect("source timestamp should be valid"),
|
||||
),
|
||||
size: source.size,
|
||||
etag: Some(source.etag.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(local_object_matches_transition_source(&exact, &source));
|
||||
|
||||
let mut changed = exact.clone();
|
||||
changed.version_id = Some(Uuid::new_v4());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.data_dir = Some(Uuid::new_v4());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.mod_time = changed.mod_time.map(|value| value + Duration::from_nanos(1));
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact.clone();
|
||||
changed.size += 1;
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
changed = exact;
|
||||
changed.etag = Some("different-etag".to_string());
|
||||
assert!(!local_object_matches_transition_source(&changed, &source));
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
@@ -2629,57 +2384,6 @@ mod tests {
|
||||
assert_eq!(transaction.state, TransitionTransactionState::Committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_state_sequence_is_distinguishable_and_keeps_legacy_edges_strict() {
|
||||
let init = TransitionTransactionInit {
|
||||
deployment_id: Uuid::new_v4(),
|
||||
transaction_id: Uuid::new_v4(),
|
||||
owner_epoch: Uuid::new_v4(),
|
||||
write_id: Uuid::new_v4(),
|
||||
source: source_identity(TransitionSourceVersionMode::Versioned),
|
||||
tier_name: "warm-tier".to_string(),
|
||||
backend_fingerprint: BACKEND_FINGERPRINT,
|
||||
not_after_unix_nanos: 1_780_000_000_000_000_000,
|
||||
};
|
||||
let mut compact = TransitionTransaction::new_compact(init).expect("compact transaction should be created");
|
||||
assert_eq!(compact.state, TransitionTransactionState::UploadOutcomeUnknown);
|
||||
assert_eq!(compact.revision, 1);
|
||||
let remote_version = TransitionRemoteVersion::versioned(Uuid::new_v4().to_string());
|
||||
let fence = compact
|
||||
.advance(
|
||||
compact.fence(),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
Some(remote_version.clone()),
|
||||
)
|
||||
.expect("compact upload should persist its exact candidate at the local commit fence");
|
||||
assert_eq!(fence.revision, 2);
|
||||
assert_eq!(compact.remote_version, remote_version);
|
||||
assert_eq!(compact.state, TransitionTransactionState::LocalCommitStarted);
|
||||
let encoded = compact
|
||||
.encode()
|
||||
.expect("compact transaction should encode as v1-compatible bytes");
|
||||
assert_eq!(
|
||||
TransitionTransaction::decode(compact.transaction_id, &encoded).expect("compact transaction should decode"),
|
||||
compact
|
||||
);
|
||||
|
||||
let mut legacy_unknown = new_transaction();
|
||||
legacy_unknown
|
||||
.advance(legacy_unknown.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("legacy transaction should persist its pre-upload fence");
|
||||
assert!(matches!(
|
||||
legacy_unknown.advance(
|
||||
legacy_unknown.fence(),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
Some(TransitionRemoteVersion::unversioned()),
|
||||
),
|
||||
Err(TransitionTransactionError::InvalidStateChange {
|
||||
from: TransitionTransactionState::UploadOutcomeUnknown,
|
||||
to: TransitionTransactionState::LocalCommitStarted,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_pending_requires_exact_proof_and_state_specific_decision() {
|
||||
let mut transaction = new_transaction();
|
||||
|
||||
@@ -66,7 +66,6 @@ pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge;
|
||||
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
|
||||
pub use replication_object_bridge::ReplicationObjectBridge;
|
||||
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
|
||||
pub(crate) use replication_object_decision_boundary::replication_etags_match;
|
||||
pub use replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
@@ -89,6 +88,5 @@ pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub use replication_target_boundary::VersionIdentityCapability;
|
||||
pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_filemeta::ObjectPartInfo;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::ReplicationMultipartPlanError;
|
||||
pub use rustfs_replication::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
|
||||
@@ -3079,11 +3079,7 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
|
||||
let rcfg = match ReplicationMetadataStore::optional_replication_config(bucket).await {
|
||||
Ok(Some(config)) => Some(config),
|
||||
// A bucket without a configuration still owes its pending purges an
|
||||
// answer: the delete worker finishes them locally as abandoned, which
|
||||
// is what makes the bucket deletable again (rustfs/backlog#2340).
|
||||
Ok(None) if !oi.version_purge_status.is_empty() => None,
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => return ReplicationQueueAdmission::Skipped,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
@@ -3133,7 +3129,7 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
};
|
||||
|
||||
let rcfg_wrapper = ReplicationConfig::new(rcfg, tgts);
|
||||
let rcfg_wrapper = ReplicationConfig::new(Some(rcfg), tgts);
|
||||
queue_replication_heal_internal(bucket, oi, rcfg_wrapper, retry_count)
|
||||
.await
|
||||
.admission
|
||||
@@ -3179,11 +3175,7 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
};
|
||||
}
|
||||
|
||||
// Without a configuration or targets there is nothing to replicate —
|
||||
// except a version purge the bucket still owes: its stored decision names
|
||||
// the targets, and the delete worker settles the ones no longer
|
||||
// configured as abandoned (rustfs/backlog#2340).
|
||||
if (rcfg.config.is_none() || rcfg.remotes.is_none()) && oi.version_purge_status.is_empty() {
|
||||
if rcfg.config.is_none() || rcfg.remotes.is_none() {
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
@@ -3228,15 +3220,12 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
}
|
||||
ReplicationHealQueueAction::QueueDelete(dv) => {
|
||||
// A purge the peer denied under object lock cannot succeed until
|
||||
// the lock lapses (#6850), and one whose replica cannot be told
|
||||
// apart on a target that mints its own version ids cannot
|
||||
// succeed until the ledger or an operator resolves it
|
||||
// (rustfs/backlog#2340); requeuing either every heal cycle only
|
||||
// the lock lapses (#6850); requeuing it every heal cycle only
|
||||
// burns bandwidth and failure counters. The backoff expires on
|
||||
// its own, so the purge is probed again — and converges — once
|
||||
// the condition has a chance of being over.
|
||||
// the retention window has a chance of being over.
|
||||
if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
|
||||
&& super::replication_resyncer::purge_backoff_active(&dv)
|
||||
&& super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
|
||||
{
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,7 +38,7 @@ use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
|
||||
ReplicaLocation, S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
@@ -48,7 +48,6 @@ pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
};
|
||||
pub use rustfs_replication::{VersionIdentityCapability, version_identity_capability_from_put};
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -193,14 +192,6 @@ impl ReplicationTargetStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn version_identity_capability(arn: &str) -> VersionIdentityCapability {
|
||||
BucketTargetSys::get().version_identity_capability(arn)
|
||||
}
|
||||
|
||||
pub(crate) fn record_version_identity_capability(arn: &str, capability: VersionIdentityCapability) {
|
||||
BucketTargetSys::get().record_version_identity_capability(arn, capability)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -257,16 +248,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string());
|
||||
}
|
||||
|
||||
// Older transformed objects can have physical parts without logical part
|
||||
// lengths. Keep their existing whole-object transport: physical sizes are
|
||||
// not plaintext boundaries for a multipart replication read.
|
||||
let legacy_single_put = object_info.etag.as_deref().is_none_or(|etag| etag.len() == 32);
|
||||
let base_is_multipart = object_info.is_multipart()
|
||||
&& !(legacy_single_put
|
||||
&& object_info.parts.len() > 1
|
||||
&& (object_info.is_compressed() || object_info.is_encrypted())
|
||||
&& object_info.parts.iter().any(|part| part.actual_size <= 0));
|
||||
let mut is_multipart = base_is_multipart;
|
||||
let mut is_multipart = object_info.is_multipart();
|
||||
|
||||
if let Some(checksum_data) = &object_info.checksum
|
||||
&& !checksum_data.is_empty()
|
||||
@@ -277,8 +259,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
} else if object_info.is_encrypted() {
|
||||
// Encrypted checksums cannot be exposed as plaintext headers, and
|
||||
// decrypt_checksums reports is_multipart=false for them (a value
|
||||
// the response path relies on). Keep the transport selected from
|
||||
// the object's layout and readable part boundaries.
|
||||
// the response path relies on). Keep the object's own multipart
|
||||
// flag so encrypted objects stay on the multipart route.
|
||||
} else {
|
||||
let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
|
||||
// The checksum record describes how the *checksum* is composed,
|
||||
@@ -286,37 +268,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
// MULTIPART flag even on a multipart upload, so trusting it here
|
||||
// routed a 768-part object through a single PutObject and the
|
||||
// target rejected the 6 GiB body with EntityTooLarge
|
||||
// (rustfs#6825). The usable part layout is the authority: the
|
||||
// (rustfs#6825). The object's own shape is the authority: the
|
||||
// record may only add multipart-ness, never take it away.
|
||||
is_multipart = base_is_multipart || checksum_record_is_multipart;
|
||||
is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
|
||||
|
||||
if !base_is_multipart
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key != AMZ_CHECKSUM_TYPE {
|
||||
meta.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !object_info.is_multipart()
|
||||
&& checksum_meta
|
||||
.get(AMZ_CHECKSUM_TYPE)
|
||||
.is_some_and(|value| value == AMZ_CHECKSUM_TYPE_FULL_OBJECT)
|
||||
{
|
||||
is_multipart = false;
|
||||
}
|
||||
|
||||
// The record keys each checksum by algorithm name ("CRC32"); the
|
||||
// target only reads `x-amz-checksum-<algorithm>`. Inserting the bare
|
||||
// name here made `PutObjectOptions::header()` send it as user
|
||||
// metadata (`x-amz-meta-crc32`), so no replica ever carried the
|
||||
// source checksum (rustfs/backlog#2340). The object-level record
|
||||
// describes one PUT body: a multipart replica is rebuilt part by
|
||||
// part, and its CreateMultipartUpload must not announce a checksum
|
||||
// the parts do not carry, so the record is forwarded on the
|
||||
// single-PUT route only (MinIO `getCRCMeta` parity).
|
||||
if !is_multipart {
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key == AMZ_CHECKSUM_TYPE {
|
||||
continue;
|
||||
}
|
||||
if let Some(header) = rustfs_rio::ChecksumType::from_string(key).key() {
|
||||
meta.insert(header.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -548,7 +516,6 @@ fn is_standard_header(key: &str) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ObjectPartInfo;
|
||||
use super::*;
|
||||
use aws_smithy_types::DateTime;
|
||||
use rustfs_replication::content_matches_by_etag;
|
||||
@@ -583,109 +550,6 @@ mod tests {
|
||||
checksum.to_bytes(&combined)
|
||||
}
|
||||
|
||||
fn replication_route_metadata() -> [(&'static str, Arc<HashMap<String, String>>); 4] {
|
||||
let mut compressed = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut compressed, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
[
|
||||
("plain", Arc::new(HashMap::new())),
|
||||
("compressed", Arc::new(compressed)),
|
||||
(
|
||||
"encrypted",
|
||||
Arc::new(HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
(
|
||||
"ssec",
|
||||
Arc::new(HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())])),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn replication_route_object(
|
||||
etag: Option<&str>,
|
||||
actual_sizes: [i64; 3],
|
||||
metadata: Arc<HashMap<String, String>>,
|
||||
) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
size: 48,
|
||||
actual_size: 12,
|
||||
user_defined: metadata,
|
||||
parts: Arc::new(
|
||||
actual_sizes
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, actual_size)| ObjectPartInfo {
|
||||
number: index + 1,
|
||||
size: 16,
|
||||
actual_size,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transformed_single_put_parts_keep_the_previous_replication_route() {
|
||||
let [_, (_, compressed), (_, encrypted), (_, ssec)] = replication_route_metadata();
|
||||
let cases = [
|
||||
(
|
||||
"compressed middle zero",
|
||||
compressed.clone(),
|
||||
Some("0123456789abcdef0123456789abcdef"),
|
||||
[4, 0, 4],
|
||||
),
|
||||
("compressed tail unknown", compressed, None, [4, 4, -1]),
|
||||
(
|
||||
"encrypted middle unknown",
|
||||
encrypted,
|
||||
Some("gggggggggggggggggggggggggggggggg"),
|
||||
[4, -1, 4],
|
||||
),
|
||||
("ssec tail zero", ssec.clone(), None, [4, 4, 0]),
|
||||
("ssec middle unknown", ssec, Some("gggggggggggggggggggggggggggggggg"), [4, -1, 4]),
|
||||
];
|
||||
for (name, metadata, etag, actual_sizes) in cases {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let mut object_info = replication_route_object(etag, actual_sizes, metadata.clone());
|
||||
object_info.checksum = checksum;
|
||||
assert!(object_info.is_multipart(), "{name}: physical parts remain visible to metadata APIs");
|
||||
assert!(object_info.is_compressed() || object_info.is_encrypted());
|
||||
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("legacy transformed put options");
|
||||
assert!(
|
||||
!is_multipart,
|
||||
"{name}: unknown logical part sizes must preserve the old whole-object route"
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
if metadata.contains_key(SSEC_ALGORITHM_HEADER) {
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_SSEC_CRC).is_some(),
|
||||
object_info.checksum.is_some(),
|
||||
"SSE-C checksums retain their raw passthrough transport"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positive_part_sizes_and_legacy_multipart_etags_keep_the_replication_route() {
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
for (etag, actual_sizes) in [
|
||||
("0123456789abcdef0123456789abcdef", [4, 4, 4]),
|
||||
("0123456789abcdef0123456789abcdef-3", [4, 0, -1]),
|
||||
] {
|
||||
let mut object_info = replication_route_object(Some(etag), actual_sizes, metadata.clone());
|
||||
object_info.checksum = Some(full_object_multipart_checksum_record());
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("multipart put options");
|
||||
assert!(is_multipart, "{name}/{etag}: usable sizes and old multipart ETags must retain MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
|
||||
// rustfs#6825: a 768-part upload was replicated with a single
|
||||
@@ -718,36 +582,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_multipart_parts_keep_the_replication_route_without_a_multipart_etag() {
|
||||
for etag in [Some("0123456789abcdef0123456789abcdef"), None] {
|
||||
for checksum in [None, Some(full_object_multipart_checksum_record())] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
checksum,
|
||||
parts: Arc::new(
|
||||
(1..=2)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let (options, is_multipart) =
|
||||
replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(
|
||||
is_multipart,
|
||||
"stored parts must retain multipart routing: etag={etag:?}, checksum={:?}",
|
||||
object_info.checksum
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, etag.unwrap_or_default());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
|
||||
// The mirror of the rustfs#6825 guard: an object stored as one PUT
|
||||
@@ -758,10 +592,6 @@ mod tests {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -798,19 +628,6 @@ mod tests {
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
|
||||
|
||||
for (name, metadata) in replication_route_metadata() {
|
||||
let mut legacy = replication_route_object(Some("0123456789abcdef0123456789abcdef"), [4, 0, 4], metadata);
|
||||
legacy.checksum = Some(checksum.to_bytes(&combined));
|
||||
let (_, record_is_multipart) = legacy.decrypt_checksums(0, &HeaderMap::new()).expect("decode checksum");
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &legacy).expect("legacy checksum put options");
|
||||
if legacy.is_encrypted() {
|
||||
assert!(!is_multipart, "{name}: encrypted checksum records must not change the old transport");
|
||||
} else {
|
||||
assert!(record_is_multipart, "the composite checksum must carry its own multipart signal");
|
||||
assert!(is_multipart, "{name}: a composite record can still promote the legacy route to MPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1491,63 +1308,12 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
let (opts, _is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
|
||||
assert!(!is_multipart, "{name}: a single-part checksum record must keep the single-PUT route");
|
||||
let header = ty.key().expect("every forwarded algorithm has an x-amz-checksum header");
|
||||
assert_eq!(
|
||||
opts.user_metadata.get(header),
|
||||
opts.user_metadata.get(name),
|
||||
Some(&checksum.encoded),
|
||||
"replication must forward the {name} checksum as the {header} header"
|
||||
);
|
||||
assert!(
|
||||
!opts.user_metadata.contains_key(name),
|
||||
"{name}: the bare algorithm name would leave as x-amz-meta user metadata"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The object-level record of a multipart upload (composite or full-object)
|
||||
/// must not become a PutObject checksum header: the replica is rebuilt
|
||||
/// through CreateMultipartUpload/UploadPart, and a checksum announced there
|
||||
/// that the parts do not carry would be rejected by the target.
|
||||
#[test]
|
||||
fn replication_put_object_options_keeps_multipart_checksum_records_off_the_wire() {
|
||||
let mut composite_type = rustfs_rio::ChecksumType::from_string("crc32");
|
||||
composite_type
|
||||
.merge(rustfs_rio::ChecksumType::MULTIPART)
|
||||
.merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
|
||||
let mut combined = Vec::new();
|
||||
for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
|
||||
let part_checksum =
|
||||
rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
|
||||
combined.extend_from_slice(part_checksum.raw.as_slice());
|
||||
}
|
||||
let composite = rustfs_rio::Checksum::new_from_data(composite_type, &combined)
|
||||
.expect("composite checksum")
|
||||
.to_bytes(&combined);
|
||||
|
||||
for (label, checksum, etag) in [
|
||||
("composite", composite, "0123456789abcdef0123456789abcdef-2"),
|
||||
(
|
||||
"full-object",
|
||||
full_object_multipart_checksum_record(),
|
||||
"0123456789abcdef0123456789abcdef-3",
|
||||
),
|
||||
] {
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some(etag.to_string()),
|
||||
checksum: Some(checksum),
|
||||
..Default::default()
|
||||
};
|
||||
let (opts, is_multipart) = replication_put_object_options("", &object_info).expect("build replication put options");
|
||||
assert!(is_multipart, "{label}: a multipart object must keep the multipart route");
|
||||
assert!(
|
||||
opts.user_metadata
|
||||
.keys()
|
||||
.all(|key| !key.starts_with("x-amz-checksum-") && key != "CRC32"),
|
||||
"{label}: no object-level checksum may reach the target's CreateMultipartUpload: {:?}",
|
||||
opts.user_metadata
|
||||
"replication must forward the {name} checksum into user_metadata identically to the classic algorithms"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,8 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
|
||||
ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
pub use peer_s3_client::{
|
||||
|
||||
@@ -49,11 +49,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
|
||||
SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
|
||||
TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
|
||||
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -93,7 +92,6 @@ const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -194,102 +192,12 @@ pub struct ScannerPeerActivity {
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageSnapshot {
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub generation: u64,
|
||||
pub pending_bucket_count: u64,
|
||||
pub protocol_version: u32,
|
||||
pub complete: bool,
|
||||
pub buckets: BTreeMap<String, ScannerPeerDirtyUsageBucket>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerDirtyUsageBucket {
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerScopedDirtyUsageAckEntry {
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerDirtyUsageAcknowledgement {
|
||||
Generation {
|
||||
host: String,
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
},
|
||||
Scoped {
|
||||
host: String,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payloads(
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<Vec<ScannerScopedDirtyUsageAckRequest>> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
if entries.is_empty() {
|
||||
return Err(Error::other("scoped dirty usage acknowledgement entries must be nonempty"));
|
||||
}
|
||||
|
||||
let mut payloads = Vec::with_capacity(entries.len().div_ceil(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize));
|
||||
let mut batch = Vec::with_capacity(SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
for entry in entries {
|
||||
batch.push(ScannerScopedDirtyUsageEntry {
|
||||
bucket: entry.bucket,
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
generation: entry.generation,
|
||||
});
|
||||
if batch.len() == SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(
|
||||
&owner_id,
|
||||
&instance_id,
|
||||
probe_only,
|
||||
std::mem::take(&mut batch),
|
||||
)?);
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
payloads.push(scanner_scoped_dirty_usage_ack_payload(&owner_id, &instance_id, probe_only, batch)?);
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_payload(
|
||||
owner_id: &str,
|
||||
instance_id: &str,
|
||||
probe_only: bool,
|
||||
entries: Vec<ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<ScannerScopedDirtyUsageAckRequest> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
instance_id: instance_id.to_string(),
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only,
|
||||
entries,
|
||||
};
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
pub buckets: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
|
||||
@@ -443,11 +351,6 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if !scanner_instance_id_is_valid(&response.instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
|
||||
}
|
||||
let owner_id = Uuid::parse_str(&response.owner_id)
|
||||
.ok()
|
||||
.filter(|owner_id| !owner_id.is_nil())
|
||||
.map(|owner_id| owner_id.to_string())
|
||||
.ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot owner"))?;
|
||||
if response.generation == u64::MAX {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
|
||||
}
|
||||
@@ -483,14 +386,9 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
if bucket.generation == 0 || bucket.generation > response.generation {
|
||||
return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
|
||||
}
|
||||
Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.ok()
|
||||
.filter(|bucket_incarnation| !bucket_incarnation.is_nil())
|
||||
.ok_or_else(|| Error::other("peer scanner dirty usage snapshot contains an invalid bucket incarnation"))?;
|
||||
}
|
||||
|
||||
Ok(ScannerPeerDirtyUsageSnapshot {
|
||||
owner_id,
|
||||
instance_id: response.instance_id,
|
||||
generation: response.generation,
|
||||
pending_bucket_count: response.pending_bucket_count,
|
||||
@@ -499,16 +397,7 @@ fn decode_scanner_dirty_usage_snapshot_with_verifier(
|
||||
buckets: response
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| {
|
||||
(
|
||||
bucket.bucket,
|
||||
ScannerPeerDirtyUsageBucket {
|
||||
bucket_incarnation: Uuid::from_slice(bucket.bucket_incarnation.as_ref())
|
||||
.expect("bucket incarnation was validated"),
|
||||
generation: bucket.generation,
|
||||
},
|
||||
)
|
||||
})
|
||||
.map(|bucket| (bucket.bucket, bucket.generation))
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
@@ -1799,24 +1688,6 @@ impl PeerRestClient {
|
||||
Ok((self.topology_member.clone(), supported_version, epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn probe_transition_transaction_compaction(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::transition_transaction_compaction_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
|
||||
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
|
||||
@@ -2197,10 +2068,19 @@ impl PeerRestClient {
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id, true, entries)?;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
@@ -2208,106 +2088,26 @@ impl PeerRestClient {
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical =
|
||||
canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
if !response.supported {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<ScannerScopedDirtyUsageAckEntry>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?;
|
||||
let ack_attempt = async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
for payload in payloads {
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| !response.supported
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = match timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, ack_attempt).await {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
Err(_) => {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
Err(Error::other("scoped dirty usage acknowledgement deadline elapsed"))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let activity = self.scanner_scoped_dirty_usage_activity_confirmation().await?;
|
||||
if activity.instance_id == instance_id {
|
||||
Ok(activity)
|
||||
} else {
|
||||
Err(Error::other(
|
||||
"scoped dirty usage acknowledgement peer restarted before activity confirmation",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if Self::is_network_like_error(&err) {
|
||||
self.prepare_retry_with_timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT)
|
||||
.await;
|
||||
}
|
||||
match self.scanner_scoped_dirty_usage_activity_confirmation().await {
|
||||
Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity),
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_scoped_dirty_usage_activity_confirmation(&self) -> Result<ScannerPeerActivity> {
|
||||
timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, self.scanner_activity())
|
||||
.await
|
||||
.map_err(|_| Error::other("scoped dirty usage activity confirmation timed out"))?
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
@@ -3036,79 +2836,16 @@ mod tests {
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
},
|
||||
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
|
||||
},
|
||||
],
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
owner_id: "33333333-3333-3333-3333-333333333333".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_payloads_split_at_protocol_limit() {
|
||||
use rustfs_protos::scoped_dirty_usage::{SCOPED_DIRTY_USAGE_MAX_ENTRIES, canonical_scoped_dirty_usage_request};
|
||||
|
||||
let entries = (0..=SCOPED_DIRTY_USAGE_MAX_ENTRIES)
|
||||
.map(|index| ScannerScopedDirtyUsageAckEntry {
|
||||
bucket: format!("bucket-{index:02}"),
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111),
|
||||
generation: 9,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let payloads = scanner_scoped_dirty_usage_ack_payloads(
|
||||
"33333333-3333-3333-3333-333333333333".to_string(),
|
||||
"0123456789abcdef0123456789abcdef".to_string(),
|
||||
false,
|
||||
entries,
|
||||
)
|
||||
.expect("33 entries should split into valid scoped dirty usage requests");
|
||||
|
||||
assert_eq!(payloads.len(), 2);
|
||||
assert_eq!(payloads[0].entries.len(), SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize);
|
||||
assert_eq!(payloads[1].entries.len(), 1);
|
||||
assert_eq!(payloads[0].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-00"));
|
||||
assert_eq!(payloads[0].entries.last().map(|entry| entry.bucket.as_str()), Some("bucket-31"));
|
||||
assert_eq!(payloads[1].entries.first().map(|entry| entry.bucket.as_str()), Some("bucket-32"));
|
||||
for payload in payloads {
|
||||
canonical_scoped_dirty_usage_request(&payload).expect("each split scoped ACK payload should be canonical");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: Some([1; 32]),
|
||||
data_movement_active: Some(false),
|
||||
dirty_usage_generation: Some(9),
|
||||
dirty_usage_pending: pending,
|
||||
movement_generation: Some(1),
|
||||
publication_blocked: Some(false),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("0123456789abcdef0123456789abcdef", Some(true)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
assert!(!scanner_scoped_dirty_usage_ack_reconciled(
|
||||
&activity("fedcba9876543210fedcba9876543210", Some(false)),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
|
||||
let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
|
||||
@@ -3117,18 +2854,9 @@ mod tests {
|
||||
assert_eq!(decoded.generation, 7);
|
||||
assert_eq!(decoded.pending_bucket_count, 2);
|
||||
assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
|
||||
assert_eq!(decoded.owner_id, "33333333-3333-3333-3333-333333333333");
|
||||
assert!(decoded.complete);
|
||||
assert_eq!(
|
||||
decoded.buckets.get("archive").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x11111111111111111111111111111111))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("archive").map(|bucket| bucket.generation), Some(3));
|
||||
assert_eq!(
|
||||
decoded.buckets.get("photos").map(|bucket| bucket.bucket_incarnation),
|
||||
Some(Uuid::from_u128(0x22222222222222222222222222222222))
|
||||
);
|
||||
assert_eq!(decoded.buckets.get("photos").map(|bucket| bucket.generation), Some(7));
|
||||
assert_eq!(decoded.buckets.get("archive"), Some(&3));
|
||||
assert_eq!(decoded.buckets.get("photos"), Some(&7));
|
||||
|
||||
let overflow_count =
|
||||
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
|
||||
@@ -3179,14 +2907,6 @@ mod tests {
|
||||
empty_bucket.buckets[0].bucket.clear();
|
||||
cases.push((empty_bucket, "empty bucket name"));
|
||||
|
||||
let mut invalid_owner = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_owner.owner_id.clear();
|
||||
cases.push((invalid_owner, "snapshot owner"));
|
||||
|
||||
let mut invalid_incarnation = test_scanner_dirty_usage_snapshot_response();
|
||||
invalid_incarnation.buckets[0].bucket_incarnation = Uuid::nil().as_bytes().to_vec().into();
|
||||
cases.push((invalid_incarnation, "bucket incarnation"));
|
||||
|
||||
let mut partial = test_scanner_dirty_usage_snapshot_response();
|
||||
partial.complete = false;
|
||||
cases.push((partial, "entry-limit overflow"));
|
||||
@@ -3199,7 +2919,6 @@ mod tests {
|
||||
.map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
|
||||
bucket: format!("bucket-{index:04}"),
|
||||
generation: 1,
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
})
|
||||
.collect(),
|
||||
..test_scanner_dirty_usage_snapshot_response()
|
||||
|
||||
@@ -449,14 +449,6 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -465,6 +457,14 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
@@ -476,18 +476,6 @@ where
|
||||
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata_opts<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
/// Read an existing config object without treating an empty payload as absent.
|
||||
/// Callers that validate their own payload format need to distinguish corruption
|
||||
/// from `ConfigNotFound`.
|
||||
|
||||
@@ -3385,7 +3385,7 @@ pub(crate) async fn acquire_pool_activation_fleet_proof(
|
||||
.ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
|
||||
}
|
||||
|
||||
pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||
// Save-stage helpers add context by formatting the original error, so the
|
||||
// marker may be nested in the display string. Restrict matching to the
|
||||
// `Error::other` I/O shape used by this activation path.
|
||||
@@ -5108,7 +5108,49 @@ async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<Po
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await
|
||||
let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
|
||||
let batch = uuid::Uuid::new_v4();
|
||||
for (pool, read) in reads.iter().enumerate() {
|
||||
let mut observation = serde_json::json!({
|
||||
"kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
|
||||
"cas": match &read.cas {
|
||||
PoolMetaCasToken::Missing => "missing",
|
||||
PoolMetaCasToken::Existing(_) => "existing",
|
||||
PoolMetaCasToken::Unsafe => "unsafe",
|
||||
},
|
||||
"etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
|
||||
});
|
||||
match &read.replica {
|
||||
PoolMetaReplica::Valid {
|
||||
raw,
|
||||
canonical,
|
||||
meta,
|
||||
revision,
|
||||
committed,
|
||||
..
|
||||
} => {
|
||||
observation["state"] = serde_json::json!("valid");
|
||||
observation["committed"] = serde_json::json!(committed);
|
||||
observation["version"] = serde_json::json!(revision.version);
|
||||
observation["cluster_id"] = serde_json::json!(revision.cluster_id);
|
||||
observation["epoch"] = serde_json::json!(revision.epoch);
|
||||
observation["generation"] = serde_json::json!(revision.generation);
|
||||
observation["transaction_id"] = serde_json::json!(revision.transaction_id);
|
||||
observation["pool_count"] = serde_json::json!(meta.pools.len());
|
||||
observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
|
||||
observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
|
||||
}
|
||||
PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
|
||||
PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
|
||||
PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
|
||||
PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
|
||||
}
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
}
|
||||
reads
|
||||
}
|
||||
|
||||
fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
|
||||
@@ -5137,23 +5179,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn select_pool_meta_replicas_for_read_probe<R>(
|
||||
write_state: &PoolMetaWriteState,
|
||||
replicas: Vec<R>,
|
||||
operation: &str,
|
||||
) -> Result<PoolMetaSelection>
|
||||
where
|
||||
R: Into<PoolMetaReplicaRead>,
|
||||
{
|
||||
// Read-only planning probes must fail the current request on unsafe pool
|
||||
// metadata, but they must not permanently poison the shared writer gate.
|
||||
let mut probe_state = write_state.clone();
|
||||
let selection = select_pool_meta_replicas_observing(&mut probe_state, replicas)?;
|
||||
probe_state.observe_replicas(selection.replica_state);
|
||||
probe_state.ensure_write_safe(operation)?;
|
||||
Ok(selection)
|
||||
}
|
||||
|
||||
async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -5173,19 +5198,6 @@ where
|
||||
select_pool_meta_replicas_observing(write_state, replicas)
|
||||
}
|
||||
|
||||
async fn load_pool_meta_replicas_for_read_probe<S>(
|
||||
pools: Vec<Arc<S>>,
|
||||
no_lock: bool,
|
||||
write_state: &PoolMetaWriteState,
|
||||
operation: &str,
|
||||
) -> Result<PoolMetaSelection>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let replicas = read_pool_meta_replicas(pools, no_lock).await;
|
||||
select_pool_meta_replicas_for_read_probe(write_state, replicas, operation)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedPoolMetaV3 {
|
||||
@@ -5510,6 +5522,60 @@ fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
struct StartupCasObservation {
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: Vec<usize>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
tokio::task_local! {
|
||||
static STARTUP_CAS_OBSERVATION: StartupCasObservation;
|
||||
}
|
||||
|
||||
// This scope follows only the directly polled startup future. Spawned work
|
||||
// does not inherit it; receiver evidence retains its existing RPC tuple.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
|
||||
attempt: uuid::Uuid,
|
||||
phase: &'static str,
|
||||
pools: &[Arc<S>],
|
||||
future: F,
|
||||
) -> F::Output {
|
||||
STARTUP_CAS_OBSERVATION
|
||||
.scope(
|
||||
StartupCasObservation {
|
||||
attempt,
|
||||
phase,
|
||||
// These identities are never dereferenced or logged. The
|
||||
// caller and operation keep the same pool Arcs alive.
|
||||
pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
|
||||
},
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
// Direct JSON diagnostics are independent of the startup tracing subscriber.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
|
||||
let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
|
||||
.ok()
|
||||
.and_then(|value| uuid::Uuid::parse_str(&value).ok())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
observation["nonce"] = serde_json::json!(nonce);
|
||||
observation["pid"] = serde_json::json!(std::process::id());
|
||||
let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
observation["attempt"] = serde_json::json!(scope.attempt);
|
||||
observation["startup_phase"] = serde_json::json!(scope.phase);
|
||||
});
|
||||
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
|
||||
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
|
||||
}
|
||||
|
||||
async fn save_pool_meta_object_cas<S>(
|
||||
pool: Arc<S>,
|
||||
object: &str,
|
||||
@@ -5530,13 +5596,43 @@ where
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_options(&mut opts);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
|
||||
serde_json::json!({
|
||||
"kind": "cas", "object": object, "phase": phase,
|
||||
"pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
|
||||
scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
|
||||
}).ok().flatten(),
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
|
||||
"if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
|
||||
"if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
|
||||
"tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
|
||||
"no_lock": opts.no_lock,
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let object_info = result?;
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
let result = result.and_then(|object_info| {
|
||||
fence.ensure_held()?;
|
||||
Ok(object_info)
|
||||
});
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(mut observation) = observation {
|
||||
observation["ok"] = serde_json::json!(result.is_ok());
|
||||
observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
|
||||
observation["mod_time"] = serde_json::json!(
|
||||
result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|info| info.mod_time)
|
||||
.map(|time| time.unix_timestamp_nanos().to_string())
|
||||
);
|
||||
observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
|
||||
startup_cas_test_observe(observation);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn persist_pool_meta_identity<S>(
|
||||
@@ -6835,6 +6931,13 @@ impl PoolMeta {
|
||||
};
|
||||
if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
|
||||
persist_pool_meta_identity(pools, write_state, true, fence).await?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "confirmed", "object": POOL_META_NAME,
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
|
||||
"generation": confirmed.revision.generation,
|
||||
"transaction_id": confirmed.revision.transaction_id,
|
||||
}));
|
||||
return Ok(confirmed.meta);
|
||||
}
|
||||
if !commit_succeeded {
|
||||
@@ -9027,7 +9130,9 @@ impl ECStore {
|
||||
})?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
|
||||
let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
|
||||
let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
|
||||
write_state.observe_replicas(selection.replica_state);
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
Ok((pool_meta_guard, selection.meta))
|
||||
}
|
||||
|
||||
@@ -17980,41 +18085,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_does_not_latch_writer_state() {
|
||||
let write_state = PoolMetaWriteState::default();
|
||||
select_pool_meta_replicas_for_read_probe(
|
||||
&write_state,
|
||||
vec![PoolMetaReplica::Unreadable("transient read failure".to_string())],
|
||||
"capacity probe",
|
||||
)
|
||||
.expect_err("an unreadable probe replica must fail the current admission");
|
||||
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a read-only capacity probe must not permanently latch the pool metadata writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
for set in &store.pools[1].disk_set {
|
||||
let mut disks = set.disks.write().await;
|
||||
let disk_count = disks.len();
|
||||
*disks = vec![None; disk_count];
|
||||
}
|
||||
|
||||
let mut write_state = store.pool_meta_save_gate.lock().await;
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&mut write_state, "capacity probe")
|
||||
.await
|
||||
.expect_err("an unreadable metadata replica must reject this probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a failed read-only probe must remain retryable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
|
||||
fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
|
||||
|
||||
@@ -2673,7 +2673,7 @@ mod tests {
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(should_use_multipart_data_movement(&object_info, false));
|
||||
|
||||
let single_nonstandard_part = ObjectInfo {
|
||||
@@ -3050,7 +3050,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(object_info.is_multipart());
|
||||
assert!(!object_info.is_multipart());
|
||||
assert!(object_info.parts.iter().any(|part| part.checksums.is_some()));
|
||||
let opts = data_movement_put_object_opts(&object_info, 0);
|
||||
assert!(!rustfs_utils::http::contains_key_str(&opts.user_defined, SUFFIX_PART_CHECKSUMS));
|
||||
|
||||
@@ -195,13 +195,6 @@ fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
|
||||
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
/// Artificial `disk_info` latency for tests that pin how the admin storage
|
||||
/// walk composes per-drive probe time.
|
||||
pub(crate) static DISK_INFO_PROBE_DELAY_FOR_TEST: Duration;
|
||||
}
|
||||
|
||||
fn get_drive_timeout_profile() -> DriveTimeoutProfile {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -331,46 +324,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
Box::pin(
|
||||
self.disk
|
||||
.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2043,10 +1996,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"disk_info",
|
||||
|| async {
|
||||
#[cfg(test)]
|
||||
if let Ok(delay) = DISK_INFO_PROBE_DELAY_FOR_TEST.try_with(|delay| *delay) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
|
||||
if let Some(current_disk_id) = *self.disk_id.read().await
|
||||
|
||||
+121
-835
File diff suppressed because it is too large
Load Diff
@@ -732,46 +732,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_version_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
force_del_marker: bool,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete_version(volume, path, fi, force_del_marker, opts).await;
|
||||
// This is sender lifetime only, not proof of a remote physical drain.
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.delete_with_namespace_owner(volume, path, opts, namespace_owner).await,
|
||||
Self::Remote(disk) => {
|
||||
let result = disk.delete(volume, path, opts).await;
|
||||
drop(namespace_owner);
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep local undo publication owned independently of the wrapper deadline.
|
||||
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
|
||||
pub(crate) async fn undo_write_with_namespace_owner(
|
||||
|
||||
@@ -92,9 +92,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
|
||||
#[cfg(unix)]
|
||||
static FAILURES: std::sync::LazyLock<Mutex<HashMap<PathBuf, io::ErrorKind>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
#[cfg(unix)]
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
@@ -154,19 +151,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_failure(dir: &Path, kind: io::ErrorKind) {
|
||||
FAILURES
|
||||
.lock()
|
||||
.expect("fsync dir failure hook poisoned")
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&FAILURES, dir, "fsync dir failure hook poisoned")
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
@@ -263,7 +247,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(test, not(windows)))]
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
|
||||
@@ -272,7 +256,9 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
#[cfg(test)]
|
||||
Rollback,
|
||||
#[cfg(test)]
|
||||
DirFsync,
|
||||
}
|
||||
|
||||
@@ -288,6 +274,7 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
install_at(Stage::PreparedRename, path, hook)
|
||||
}
|
||||
@@ -304,45 +291,50 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
|
||||
#[cfg(test)]
|
||||
static RENAME_DESTINATIONS: LazyLock<Mutex<HashMap<PathBuf, RenameDestinationHook>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
/// Controlled application-test pause at an existing physical executor boundary.
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
pub struct LocalPublicationPause {
|
||||
_hook: prepared_publication_test_hooks::Guard,
|
||||
entered: oneshot::Receiver<()>,
|
||||
_release: std::sync::mpsc::Sender<()>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RenameDestinationGuard(PathBuf);
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for RenameDestinationGuard {
|
||||
fn drop(&mut self) {
|
||||
RENAME_DESTINATIONS.lock().remove(&self.0);
|
||||
}
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
impl LocalPublicationPause {
|
||||
pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result<Self> {
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local(volume, path)
|
||||
.ok_or(DiskError::DiskNotFound)??;
|
||||
let stage = match stage {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
};
|
||||
let (entered_tx, entered) = oneshot::channel();
|
||||
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
Ok(Self {
|
||||
_hook: hook,
|
||||
entered,
|
||||
_release: release,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn observe_rename_destination(source: &Path, hook: impl FnOnce(&Path) + Send + 'static) -> RenameDestinationGuard {
|
||||
assert!(
|
||||
RENAME_DESTINATIONS
|
||||
.lock()
|
||||
.insert(source.to_path_buf(), Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
RenameDestinationGuard(source.to_path_buf())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn drain_namespace_key(path: &Path) {
|
||||
drop(super::acquire_namespace_mutation_lease(path).await);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn run_rename_destination(source: &Path, destination: &Path) {
|
||||
let hook = RENAME_DESTINATIONS.lock().remove(source);
|
||||
if let Some(hook) = hook {
|
||||
hook(destination);
|
||||
}
|
||||
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||
(&mut self.entered).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,10 +434,6 @@ pub fn fsync_dir_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dir_recorder::record(dir.as_ref());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
#[cfg(test)]
|
||||
if let Some(kind) = fsync_dir_recorder::take_failure(dir.as_ref()) {
|
||||
return Err(io::Error::from(kind));
|
||||
}
|
||||
std::fs::File::open(dir.as_ref())?.sync_all()?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
@@ -1421,7 +1409,7 @@ async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationL
|
||||
acquire_namespace_mutation_lease_with_owner(path, None).await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
|
||||
async fn acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
@@ -1976,7 +1964,7 @@ pub(crate) async fn remove_file_with_owner(
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
std::fs::remove_file(path)
|
||||
})
|
||||
@@ -1996,42 +1984,6 @@ pub(crate) async fn remove_dir_with_owner(
|
||||
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
|
||||
/// Preserve raw rename semantics while retaining a counted owner in the syscall.
|
||||
/// Unlike reliable rename, this never creates parents or retries a missing source.
|
||||
pub(in crate::disk) async fn rename_with_namespace_owner(
|
||||
src: &Path,
|
||||
dst: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::rename(src, dst).await;
|
||||
}
|
||||
let src = src.to_path_buf();
|
||||
let dst = dst.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst);
|
||||
}
|
||||
std::fs::rename(src, dst)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::create_dir_all(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
@@ -2236,7 +2188,7 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
move || {
|
||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
@@ -2367,9 +2319,7 @@ async fn reliable_rename_inner_with_lease(
|
||||
let base_dir = base_dir.clone();
|
||||
move || {
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run_rename_destination(&src_file_path, &dst_file_path);
|
||||
#[cfg(all(test, not(windows)))]
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
|
||||
@@ -2278,121 +2278,6 @@ mod tests {
|
||||
assert_eq!(read, fixture.plaintext, "SSE-C + compression full GET must reassemble all parts");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_reads_preserve_plaintext() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 0];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
for (kind, mut fixture, headers) in [
|
||||
(
|
||||
"encrypted",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"compressed and encrypted",
|
||||
compressed_encrypted_multipart_fixture(key, &part_sizes).await,
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
fixture.object_info.etag = Some(faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref()));
|
||||
assert_eq!(fixture.object_info.etag.as_ref().expect("source ETag").len(), 32);
|
||||
assert_eq!(fixture.object_info.parts.len(), 2);
|
||||
let tail = &fixture.object_info.parts[1];
|
||||
assert_eq!(tail.actual_size, 0, "{kind}: final part has no plaintext");
|
||||
if kind == "compressed" {
|
||||
assert_eq!(tail.size, 0, "unpadded compression emits no bytes for an empty part");
|
||||
} else {
|
||||
assert!(tail.size > 0, "{kind}: the empty part still has a stored frame");
|
||||
}
|
||||
let stored_size = i64::try_from(fixture.stored.len()).expect("fixture size fits i64");
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored)),
|
||||
None,
|
||||
&fixture.object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("full transformed read must include the empty tail");
|
||||
assert_eq!((offset, length), (0, stored_size), "{kind}: full read includes all stored parts");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read through the complete decoder EOF");
|
||||
assert_eq!(body, fixture.plaintext, "{kind}: no plaintext is added or lost by the empty tail");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_empty_tail_full_read_authenticates_v2_final_frame() {
|
||||
let key = [0x6Eu8; 32];
|
||||
let plaintext = legacy_fixture_part_plaintext(1, 5 * 1024 * 1024);
|
||||
let mut ciphertext = Vec::new();
|
||||
let mut parts = Vec::new();
|
||||
for (number, body) in [(1, plaintext.as_slice()), (2, b"".as_slice())] {
|
||||
let start = ciphertext.len();
|
||||
rustfs_rio::EncryptReader::new_multipart_v2(Cursor::new(body), key, LEGACY_FIXTURE_BASE_NONCE, number)
|
||||
.read_to_end(&mut ciphertext)
|
||||
.await
|
||||
.expect("encrypt a v2 fixture part with an authenticated final frame");
|
||||
parts.push(ObjectPartInfo {
|
||||
number,
|
||||
size: ciphertext.len() - start,
|
||||
actual_size: i64::try_from(body.len()).expect("fixture plaintext size fits"),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let tail_start = parts[0].size;
|
||||
assert_eq!(parts[1].actual_size, 0);
|
||||
assert!(parts[1].size > 8, "the empty final frame carries more than an END marker");
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "v2-empty-tail".to_string(),
|
||||
size: i64::try_from(ciphertext.len()).expect("fixture ciphertext size fits"),
|
||||
etag: Some(faster_hex::hex_string(Md5::digest(&plaintext).as_ref())),
|
||||
parts: Arc::new(parts),
|
||||
user_defined: Arc::new(legacy_ssec_multipart_metadata(key, plaintext.len())),
|
||||
..Default::default()
|
||||
};
|
||||
for corrupt_tail in [false, true] {
|
||||
let mut stored = ciphertext.clone();
|
||||
if corrupt_tail {
|
||||
// The v2 header is authenticated associated data, including
|
||||
// the header of a final frame containing zero plaintext.
|
||||
stored[tail_start + 5] ^= 1;
|
||||
}
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(stored)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&ssec_headers_from_key(key),
|
||||
)
|
||||
.await
|
||||
.expect("construct the full reader before consuming the final frame");
|
||||
assert_eq!((offset, length), (0, object_info.size));
|
||||
let result = tokio::io::copy(&mut reader.stream, &mut tokio::io::sink()).await;
|
||||
if corrupt_tail {
|
||||
let err = result.expect_err("EOF must authenticate the empty final frame after all plaintext is returned");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "v2 encrypted frame failed authentication");
|
||||
} else {
|
||||
assert_eq!(
|
||||
result.expect("valid empty final frame must reach EOF"),
|
||||
u64::try_from(plaintext.len()).expect("plaintext length fits")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compressed_encrypted_multipart_range_crosses_part_boundary() {
|
||||
let key_bytes = [0x6Eu8; 32];
|
||||
@@ -3771,61 +3656,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_full_read_preserves_legacy_zero_and_negative_part_sizes() {
|
||||
let key = [0x77; 32];
|
||||
let part_sizes = [5 * 1024 * 1024, 1024 * 1024];
|
||||
let encrypted = build_legacy_ssec_multipart_fixture(key, &part_sizes).await;
|
||||
// The encrypted case supplies the fixture key explicitly. This covers
|
||||
// full decrypted reads, not managed-key acquisition.
|
||||
for (kind, fixture, headers) in [
|
||||
("compressed", compressed_multipart_fixture(&part_sizes).await, HeaderMap::new()),
|
||||
(
|
||||
"encrypted with supplied key",
|
||||
CompressedMultipartFixture {
|
||||
object_info: encrypted.object_info,
|
||||
stored: encrypted.ciphertext,
|
||||
plaintext: encrypted.plaintext,
|
||||
},
|
||||
ssec_headers_from_key(key),
|
||||
),
|
||||
] {
|
||||
let source_etag = faster_hex::hex_string(Md5::digest(&fixture.plaintext).as_ref());
|
||||
assert_eq!(source_etag.len(), 32);
|
||||
assert_eq!(fixture.plaintext.len(), 6 * 1024 * 1024);
|
||||
for part_index in 0..part_sizes.len() {
|
||||
assert!(fixture.object_info.parts[part_index].actual_size > 0, "the selected part is nonempty");
|
||||
for actual_size in [0, -1] {
|
||||
let mut object_info = fixture.object_info.clone();
|
||||
object_info.etag = Some(source_etag.clone());
|
||||
Arc::make_mut(&mut object_info.parts)[part_index].actual_size = actual_size;
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
Box::new(Cursor::new(fixture.stored.clone())),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&headers,
|
||||
)
|
||||
.await
|
||||
.expect("the authoritative total size must keep full legacy reads available");
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, i64::try_from(fixture.stored.len()).expect("stored size fits"));
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("full read must reach EOF despite an unspecified per-part logical size");
|
||||
assert_eq!(
|
||||
body, fixture.plaintext,
|
||||
"{kind}: part {part_index} with actual_size={actual_size} must not lose readable data"
|
||||
);
|
||||
assert_eq!(reader.object_info.etag.as_deref(), Some(source_etag.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The physical part sizes must add up to `oi.size` for a seek to be safe;
|
||||
/// inconsistent metadata must fall back to the previous full-object read
|
||||
/// instead of scheduling an erasure read past the object end.
|
||||
|
||||
@@ -1597,7 +1597,7 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn is_multipart(&self) -> bool {
|
||||
self.parts.len() > 1 || self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
self.etag.as_ref().is_some_and(|v| v.len() != 32)
|
||||
}
|
||||
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
@@ -2235,35 +2235,6 @@ mod tests {
|
||||
}
|
||||
use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntry, TRANSITION_COMPLETE};
|
||||
|
||||
#[test]
|
||||
fn multipart_identity_uses_stored_parts_and_preserves_the_etag_fallback() {
|
||||
let plain_etag = "0123456789abcdef0123456789abcdef";
|
||||
let multipart_etag = "0123456789abcdef0123456789abcdef-1";
|
||||
for (case, part_count, etag, expected) in [
|
||||
("preserved source ETag", 2, Some(plain_etag), true),
|
||||
("missing ETag", 2, None, true),
|
||||
("ordinary PUT", 1, Some(plain_etag), false),
|
||||
("ordinary PUT without ETag", 1, None, false),
|
||||
("single-part MPU", 1, Some(multipart_etag), true),
|
||||
("legacy MPU without parts", 0, Some(multipart_etag), true),
|
||||
] {
|
||||
let object = ObjectInfo {
|
||||
etag: etag.map(str::to_string),
|
||||
parts: Arc::new(
|
||||
(1..=part_count)
|
||||
.map(|number| ObjectPartInfo {
|
||||
number,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(object.is_multipart(), expected, "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_fast_path_object(size: i64, versioned: bool) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
size,
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||
use crate::cluster::rpc::{
|
||||
PeerRestClient, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot,
|
||||
ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
|
||||
};
|
||||
use crate::diagnostics::admin_server_info::get_commit_id;
|
||||
use crate::disk::DiskAPI;
|
||||
@@ -34,11 +33,11 @@ use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{
|
||||
Arc, LazyLock, Mutex, OnceLock,
|
||||
Arc, Mutex, OnceLock,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -312,29 +311,12 @@ pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one immutable ILM recovery export.
|
||||
pub struct IlmRecoveryExportFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for emitting the compact transition-transaction
|
||||
/// state sequence. The generation permit prevents a successor proof from
|
||||
/// being published until the admitted writer has finished.
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
@@ -356,14 +338,6 @@ fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::R
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
ILM_RECOVERY_EXPORT_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -464,33 +438,6 @@ pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStat
|
||||
fleet_capability_proof_matches(remote_version_state_fleet_proof_slot(), &proof.0)
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_transition_transaction_compaction_fleet_proof() -> Option<TransitionTransactionCompactionFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let token = acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now())?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(TransitionTransactionCompactionFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub(crate) fn transition_transaction_compaction_fleet_proof_matches(
|
||||
proof: &TransitionTransactionCompactionFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = transition_transaction_compaction_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(&state, &proof.token, expected_topology, Instant::now())
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = cross_pool_fence_fleet_proof_slot()
|
||||
@@ -626,117 +573,6 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed = observe_ilm_recovery_export_fleet(expected_topology).await?;
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, &proof, expected_topology, &observed, Instant::now())
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_ilm_recovery_export_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(IlmRecoveryExportFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
pub async fn ilm_recovery_export_fleet_proof_matches(proof: &IlmRecoveryExportFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !ilm_recovery_export_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed) = observe_ilm_recovery_export_fleet(expected_topology).await else {
|
||||
return false;
|
||||
};
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
ilm_recovery_export_fleet_proof_matches_observation_at(&state, proof, expected_topology, &observed, Instant::now())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_topology_generation(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-topology-v1\0");
|
||||
hasher.update(proof.token.topology_fingerprint.as_bytes());
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_member_epochs_sha256(proof: &IlmRecoveryExportFleetProofToken) -> String {
|
||||
let encoded = serde_json::to_vec(proof.token.peer_epochs.as_ref()).expect("member epoch map is JSON encodable");
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"rustfs-ilm-recovery-export-members-v1\0");
|
||||
hasher.update(encoded);
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_local_process_epoch() -> Uuid {
|
||||
*ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn ilm_recovery_export_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &IlmRecoveryExportFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
ilm_recovery_export_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed
|
||||
}
|
||||
|
||||
async fn observe_ilm_recovery_export_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let state = ilm_recovery_export_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, Instant::now()) {
|
||||
return state.proof.as_ref().map(|proof| proof.peer_epochs.as_ref().clone());
|
||||
}
|
||||
}
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
@@ -825,7 +661,7 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
state.proof.clone()
|
||||
} else {
|
||||
Some(FleetCapabilityProof::new(
|
||||
topology.clone(),
|
||||
topology,
|
||||
Arc::new(BTreeMap::new()),
|
||||
now + Duration::from_secs(60 * 60),
|
||||
))
|
||||
@@ -858,21 +694,6 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
decommission_state.topology_conflict = false;
|
||||
decommission_state.draining_generation = None;
|
||||
decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
drop(decommission_state);
|
||||
let mut export_state = ilm_recovery_export_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !fleet_capability_proof_valid_at(export_state.proof.as_ref(), &topology, now) {
|
||||
debug_assert!(
|
||||
export_state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_none_or(|current| current.generation.is_drained())
|
||||
);
|
||||
export_state.topology_conflict = false;
|
||||
export_state.draining_generation = None;
|
||||
export_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1113,35 +934,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
|
||||
RemoteVersionStateFleetProofGuard
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionTransactionCompactionFleetProofGuard {
|
||||
fn drop(&mut self) {
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn install_transition_transaction_compaction_fleet_proof_for_test(
|
||||
topology_fingerprint: &str,
|
||||
) -> TransitionTransactionCompactionFleetProofGuard {
|
||||
let _ = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.to_string());
|
||||
let effective_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
|
||||
.get()
|
||||
.expect("transition transaction compaction test topology should be initialized");
|
||||
if let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
effective_topology,
|
||||
Ok(BTreeMap::new()),
|
||||
Instant::now(),
|
||||
) {
|
||||
panic!("test proof installation must not fail: {err}");
|
||||
}
|
||||
TransitionTransactionCompactionFleetProofGuard
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
@@ -1158,8 +950,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -1169,59 +959,29 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let notification_sys = get_global_notification_sys();
|
||||
let remote_version_state_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let cross_pool_fence_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
}
|
||||
let fence_probe = match get_global_notification_sys() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let recovery_export_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
|
||||
None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
|
||||
}
|
||||
};
|
||||
let transition_transaction_compaction_probe = async {
|
||||
match notification_sys.as_ref() {
|
||||
Some(notification_sys) => timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
|
||||
None => Err(Error::other(
|
||||
"transition transaction compaction fleet capability notification system is unavailable",
|
||||
)),
|
||||
}
|
||||
};
|
||||
let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
|
||||
remote_version_state_probe,
|
||||
cross_pool_fence_probe,
|
||||
recovery_export_probe,
|
||||
transition_transaction_compaction_probe
|
||||
);
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
@@ -1244,8 +1004,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1272,42 +1030,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
ilm_recovery_export_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
recovery_export_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "ilm_recovery_export_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
transition_transaction_compaction_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
transition_transaction_compaction_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "transition_transaction_compaction_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
@@ -1430,28 +1152,6 @@ impl NotificationSys {
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other(
|
||||
"transition transaction compaction capability fleet membership is incomplete",
|
||||
));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("transition transaction compaction capability peer is unreachable"))?;
|
||||
client
|
||||
.probe_transition_transaction_compaction(topology_fingerprint.to_string())
|
||||
.await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
|
||||
async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
|
||||
@@ -1474,46 +1174,6 @@ impl NotificationSys {
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
|
||||
async fn probe_ilm_recovery_export_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership is incomplete"));
|
||||
}
|
||||
let local_member = runtime_sources::local_node_name().await;
|
||||
if local_member.trim().is_empty() {
|
||||
return Err(Error::other("ILM recovery export local member identity is unavailable"));
|
||||
}
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, local_member.clone(), ilm_recovery_export_local_process_epoch())?;
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("ILM recovery export capability peer is unreachable"))?;
|
||||
client.probe_ilm_recovery_export(topology_fingerprint.to_string()).await
|
||||
});
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
validate_ilm_recovery_export_members(&self.peer_topology_hosts, &local_member, &peer_epochs)?;
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_ilm_recovery_export_members(
|
||||
expected_remote_members: &[String],
|
||||
local_member: &str,
|
||||
observed: &BTreeMap<String, Uuid>,
|
||||
) -> Result<()> {
|
||||
let expected = expected_remote_members
|
||||
.iter()
|
||||
.cloned()
|
||||
.chain(std::iter::once(local_member.to_string()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
if expected.len() != expected_remote_members.len().saturating_add(1) || observed.keys().ne(expected.iter()) {
|
||||
return Err(Error::other("ILM recovery export capability fleet membership does not match topology"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rolling tier activity summed over every cluster member that answered, with
|
||||
@@ -2681,70 +2341,11 @@ impl NotificationSys {
|
||||
Ok(snapshots)
|
||||
}
|
||||
|
||||
pub async fn scanner_scoped_dirty_usage_capabilities(
|
||||
&self,
|
||||
acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>,
|
||||
) -> Result<bool> {
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
ScannerDirtyUsageAcknowledgement::Generation { .. } => {
|
||||
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
|
||||
}
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
let clients = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|client| (client.grid_host.clone(), client.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
return Err(Error::other("scanner scoped dirty usage capability failed: peer is not reachable"));
|
||||
};
|
||||
futures.push(async move {
|
||||
let ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} = acknowledgement
|
||||
else {
|
||||
unreachable!("scoped acknowledgement was validated before probing");
|
||||
};
|
||||
timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
client.scanner_scoped_dirty_usage_capability(owner_id, instance_id, entries),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner scoped dirty usage capability timed out"))?
|
||||
});
|
||||
}
|
||||
|
||||
for result in join_all(futures).await {
|
||||
if !result? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
let host = match &acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
|
||||
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
for (host, instance_id, generation) in acknowledgements {
|
||||
if by_host.insert(host.clone(), (instance_id, generation)).is_some() {
|
||||
return Err(Error::other(format!("duplicate scanner dirty usage acknowledgement target: {host}")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2756,34 +2357,18 @@ impl NotificationSys {
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut failures = Vec::new();
|
||||
let mut futures = Vec::with_capacity(by_host.len());
|
||||
for (host, acknowledgement) in by_host {
|
||||
for (host, (instance_id, generation)) in by_host {
|
||||
let Some(client) = clients.get(&host).cloned() else {
|
||||
failures.push(format!("peer {host} scanner dirty usage acknowledgement failed: peer is not reachable"));
|
||||
continue;
|
||||
};
|
||||
futures.push(async move {
|
||||
let result = match acknowledgement {
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
instance_id, generation, ..
|
||||
} => {
|
||||
scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
owner_id,
|
||||
instance_id,
|
||||
entries,
|
||||
..
|
||||
} => {
|
||||
client
|
||||
.acknowledge_scanner_scoped_dirty_usage(owner_id, instance_id, entries)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let result = scanner_activity_with_timeout(
|
||||
SCANNER_ACTIVITY_PROBE_TIMEOUT,
|
||||
&host,
|
||||
client.acknowledge_scanner_dirty_usage(instance_id, generation),
|
||||
)
|
||||
.await;
|
||||
(host, result)
|
||||
});
|
||||
}
|
||||
@@ -3922,81 +3507,6 @@ mod tests {
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_member_digest_is_order_independent_and_epoch_bound() {
|
||||
let now = Instant::now();
|
||||
let local_epoch = ilm_recovery_export_local_process_epoch();
|
||||
assert!(!local_epoch.is_nil());
|
||||
assert_eq!(local_epoch, ilm_recovery_export_local_process_epoch());
|
||||
let remote_epoch = Uuid::new_v4();
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let peers = BTreeMap::from([("node-b".to_string(), remote_epoch), ("node-a".to_string(), local_epoch)]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let proof = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
let digest = ilm_recovery_export_member_epochs_sha256(&proof);
|
||||
|
||||
let changed_slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let changed = BTreeMap::from([("node-a".to_string(), local_epoch), ("node-b".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&changed_slot, "topology-a", Ok(changed), now).is_none());
|
||||
let changed_proof = {
|
||||
let state = changed_slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export")
|
||||
};
|
||||
assert_ne!(digest, ilm_recovery_export_member_epochs_sha256(&changed_proof));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_members_must_match_the_exact_topology() {
|
||||
let expected_remote = vec!["node-b".to_string()];
|
||||
let local = "node-a";
|
||||
let complete = BTreeMap::from([
|
||||
(local.to_string(), Uuid::new_v4()),
|
||||
(expected_remote[0].clone(), Uuid::new_v4()),
|
||||
]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &complete).is_ok());
|
||||
|
||||
let unexpected = BTreeMap::from([(local.to_string(), Uuid::new_v4()), ("node-c".to_string(), Uuid::new_v4())]);
|
||||
assert!(validate_ilm_recovery_export_members(&expected_remote, local, &unexpected).is_err());
|
||||
assert!(
|
||||
validate_ilm_recovery_export_members(&[local.to_string()], local, &complete).is_err(),
|
||||
"the configured remote set cannot repeat the local member"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_restart_revokes_authority_until_permit_drains() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("fresh fleet should admit export")
|
||||
};
|
||||
|
||||
let restarted = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]);
|
||||
let draining = publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted.clone()), now)
|
||||
.expect("restart must wait for the admitted export effect window");
|
||||
assert!(draining.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(!ilm_recovery_export_fleet_proof_matches_at(&state, &admitted, "topology-a", now));
|
||||
assert!(
|
||||
acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_none(),
|
||||
"successor authority must wait for the old effect window to drain"
|
||||
);
|
||||
}
|
||||
drop(admitted);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let state = slot.read().expect("export proof slot should not poison");
|
||||
assert!(acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
|
||||
let topology = "topology-a";
|
||||
@@ -4948,28 +4458,15 @@ mod tests {
|
||||
peer_topology_hosts: Vec::new(),
|
||||
};
|
||||
let missing = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
}])
|
||||
.acknowledge_scanner_dirty_usage(vec![("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7)])
|
||||
.await
|
||||
.expect_err("a missing acknowledgement target must remain pending");
|
||||
assert!(missing.to_string().contains("peer is not reachable"));
|
||||
|
||||
let duplicate = sys
|
||||
.acknowledge_scanner_dirty_usage(vec![
|
||||
ScannerDirtyUsageAcknowledgement::Generation {
|
||||
host: "peer-1".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
generation: 7,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgement::Scoped {
|
||||
host: "peer-1".to_string(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
("peer-1".to_string(), "0123456789abcdef0123456789abcdef".to_string(), 7),
|
||||
])
|
||||
.await
|
||||
.expect_err("duplicate acknowledgement targets must be rejected");
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_util;
|
||||
#[allow(clippy::module_inception, reason = "preserve the public services::tier::tier path")]
|
||||
pub mod tier;
|
||||
pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use bytes::Bytes;
|
||||
@@ -800,7 +802,7 @@ pub enum TierConfigUpdateError {
|
||||
}
|
||||
|
||||
enum TierCandidateMutation {
|
||||
Add(Box<TierConfig>, bool),
|
||||
Add(TierConfig, bool),
|
||||
Edit(String, TierCreds),
|
||||
Remove(String, bool),
|
||||
Clear(bool),
|
||||
@@ -821,7 +823,7 @@ struct PrevalidatedTierCandidateMutation {
|
||||
impl TierCandidateMutation {
|
||||
fn add(mut config: TierConfig, force: bool) -> std::result::Result<Self, AdminError> {
|
||||
normalize_s3_gcs_add_tier_name(&mut config)?;
|
||||
Ok(Self::Add(Box::new(config), force))
|
||||
Ok(Self::Add(config, force))
|
||||
}
|
||||
|
||||
fn normalize_add_tier_name(&mut self) -> std::result::Result<(), AdminError> {
|
||||
@@ -908,7 +910,7 @@ impl TierCandidateMutation {
|
||||
match self {
|
||||
Self::Add(config, force) => {
|
||||
let tier_name = config.name.clone();
|
||||
candidate.add_with_deadline(*config, force, deadline).await?;
|
||||
candidate.add_with_deadline(config, force, deadline).await?;
|
||||
Ok(Some(tier_name))
|
||||
}
|
||||
Self::Edit(tier_name, credentials) => {
|
||||
@@ -2988,7 +2990,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
|
||||
let tier_type = if wasabi_version {
|
||||
TierType::Wasabi
|
||||
} else {
|
||||
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or(match ext.tier_type {
|
||||
tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or_else(|| match ext.tier_type {
|
||||
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
|
||||
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
|
||||
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
|
||||
@@ -3370,23 +3372,28 @@ impl TierConfigMgr {
|
||||
|
||||
pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
|
||||
self.ensure_generation_is_idle(tier_name)?;
|
||||
let driver = match self.get_driver(tier_name).await {
|
||||
Ok(driver) => driver,
|
||||
Err(err) if err.code == ERR_TIER_NOT_FOUND.code => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let d = self.get_driver(tier_name).await;
|
||||
if let Err(err) = d {
|
||||
if err.code == ERR_TIER_NOT_FOUND.code {
|
||||
return Ok(());
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if !force {
|
||||
match driver.in_use().await {
|
||||
Err(err) => {
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
if let Ok(driver) = d {
|
||||
match driver.in_use().await {
|
||||
Err(err) => {
|
||||
let mut e = ERR_TIER_PERM_ERR.clone();
|
||||
e.message.push('.');
|
||||
e.message.push_str(&err.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
Ok(in_use) if in_use => {
|
||||
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(in_use) if in_use => {
|
||||
return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self.tiers.remove(tier_name);
|
||||
@@ -3395,12 +3402,21 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
|
||||
let driver = self.get_driver(tier_name).await.map_err(std::io::Error::other)?;
|
||||
check_warm_backend(Some(driver)).await.map_err(std::io::Error::other)
|
||||
let d = match self.get_driver(tier_name).await {
|
||||
Ok(d) => d,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
if let Err(err) = check_warm_backend(Some(d)).await {
|
||||
return Err(std::io::Error::other(err));
|
||||
} else {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn empty(&self) -> bool {
|
||||
self.tiers.is_empty()
|
||||
self.list_tiers().len() == 0
|
||||
}
|
||||
|
||||
pub fn tier_type(&self, tier_name: &str) -> String {
|
||||
@@ -3413,7 +3429,7 @@ impl TierConfigMgr {
|
||||
|
||||
pub fn list_tiers(&self) -> Vec<TierConfig> {
|
||||
let mut tier_cfgs = Vec::<TierConfig>::new();
|
||||
for tier in self.tiers.values() {
|
||||
for (_, tier) in self.tiers.iter() {
|
||||
let tier = tier.redacted();
|
||||
tier_cfgs.push(tier);
|
||||
}
|
||||
@@ -5442,7 +5458,7 @@ impl TierConfigMgr {
|
||||
let manager = handle.read().await;
|
||||
let published_digest = if intents
|
||||
.iter()
|
||||
.any(|recovered| recovered.intent.state == TierMutationIntentState::Committed)
|
||||
.any(|recovered| recovered.is_peer_only_terminal() && recovered.intent.state == TierMutationIntentState::Committed)
|
||||
{
|
||||
Some(tier_config_candidate_digest(&manager).map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
@@ -5452,29 +5468,18 @@ impl TierConfigMgr {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locally_published_committed_mutations = intents
|
||||
.iter()
|
||||
.filter(|recovered| {
|
||||
recovered.intent.state == TierMutationIntentState::Committed
|
||||
&& published_digest == Some(recovered.intent.candidate_digest)
|
||||
})
|
||||
.map(|recovered| recovered.intent.mutation_id)
|
||||
.collect::<HashSet<_>>();
|
||||
let mut prepared_mutation_blocks = HashMap::new();
|
||||
let mut committed_mutation_blocks: HashMap<String, HashSet<uuid::Uuid>> = HashMap::new();
|
||||
for recovered in intents {
|
||||
// A matching in-memory manager has already crossed the local
|
||||
// publication boundary. Keep replaying and durably cleaning the
|
||||
// record, but do not re-fence object operations while that
|
||||
// terminal work finishes.
|
||||
let skip_runtime_fence = locally_published_committed_mutations.contains(&recovered.intent.mutation_id)
|
||||
|| (recovered.is_peer_only_terminal()
|
||||
&& match recovered.intent.state {
|
||||
TierMutationIntentState::Aborted => true,
|
||||
TierMutationIntentState::Committed => !retain_missing_mutation_blocks,
|
||||
TierMutationIntentState::Prepared => false,
|
||||
});
|
||||
if skip_runtime_fence {
|
||||
let settled_tombstone = recovered.is_peer_only_terminal()
|
||||
&& match recovered.intent.state {
|
||||
TierMutationIntentState::Aborted => true,
|
||||
TierMutationIntentState::Committed => {
|
||||
!retain_missing_mutation_blocks || published_digest == Some(recovered.intent.candidate_digest)
|
||||
}
|
||||
TierMutationIntentState::Prepared => false,
|
||||
};
|
||||
if settled_tombstone {
|
||||
continue;
|
||||
}
|
||||
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, &recovered.intent)?;
|
||||
@@ -5487,9 +5492,6 @@ impl TierConfigMgr {
|
||||
}
|
||||
if retain_missing_mutation_blocks {
|
||||
for (tier_name, mutation_id) in &runtime.prepared_mutation_blocks {
|
||||
if locally_published_committed_mutations.contains(mutation_id) {
|
||||
continue;
|
||||
}
|
||||
match prepared_mutation_blocks.entry(tier_name.clone()) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(*mutation_id);
|
||||
@@ -5503,15 +5505,10 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
for (tier_name, mutation_ids) in &runtime.committed_mutation_blocks {
|
||||
for mutation_id in mutation_ids {
|
||||
if locally_published_committed_mutations.contains(mutation_id) {
|
||||
continue;
|
||||
}
|
||||
committed_mutation_blocks
|
||||
.entry(tier_name.clone())
|
||||
.or_default()
|
||||
.insert(*mutation_id);
|
||||
}
|
||||
committed_mutation_blocks
|
||||
.entry(tier_name.clone())
|
||||
.or_default()
|
||||
.extend(mutation_ids);
|
||||
}
|
||||
}
|
||||
let changed = runtime.prepared_mutation_blocks != prepared_mutation_blocks
|
||||
@@ -7119,8 +7116,7 @@ mod tests {
|
||||
let err = expect_decode_err(&encode_fixture(&wrong_hint));
|
||||
assert!(err.to_string().contains("inconsistent Wasabi type discriminators"), "{err}");
|
||||
|
||||
type WasabiPoisonField = (&'static str, fn(&mut ExternalTierS3));
|
||||
let poison_fields: [WasabiPoisonField; 6] = [
|
||||
let poison_fields: [(&str, fn(&mut ExternalTierS3)); 6] = [
|
||||
("storage_class", |s3| s3.storage_class = "GLACIER".to_string()),
|
||||
("aws_role", |s3| s3.aws_role = true),
|
||||
("web_identity_token", |s3| s3.aws_role_web_identity_token_file = "/tmp/token".to_string()),
|
||||
@@ -8241,11 +8237,7 @@ mod tests {
|
||||
peer_calls.clone(),
|
||||
Ok(PeerTierMutationState::Committed),
|
||||
)],
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store,
|
||||
TierCandidateMutation::Add(Box::new(tier), true),
|
||||
),
|
||||
TierConfigMgr::update_candidate_with_config_lock(&manager, store, TierCandidateMutation::Add(tier, true)),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -8284,7 +8276,7 @@ mod tests {
|
||||
let add = TIER_DRIVER_TEST_FACTORY.scope(
|
||||
factory,
|
||||
apply_tier_candidate_mutation(
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-DEADLINE")), false),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-DEADLINE"), false),
|
||||
&mut candidate,
|
||||
deadline,
|
||||
),
|
||||
@@ -9103,9 +9095,7 @@ mod tests {
|
||||
fn decode_hex_fixture(hex: &str) -> Vec<u8> {
|
||||
assert_eq!(hex.len() % 2, 0, "hex fixture must contain complete bytes");
|
||||
hex.as_bytes()
|
||||
.as_chunks::<2>()
|
||||
.0
|
||||
.iter()
|
||||
.chunks_exact(2)
|
||||
.map(|pair| {
|
||||
let pair = std::str::from_utf8(pair).expect("hex fixture should be ASCII");
|
||||
u8::from_str_radix(pair, 16).expect("hex fixture should contain only hexadecimal digits")
|
||||
@@ -11119,7 +11109,7 @@ mod tests {
|
||||
store.clone(),
|
||||
candidate,
|
||||
version,
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
update,
|
||||
None,
|
||||
)
|
||||
@@ -11308,115 +11298,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn published_dual_terminal_intent_does_not_restore_local_runtime_fence_during_replay() {
|
||||
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("published tier config fixture should persist");
|
||||
let (_, current_etag) = load_tier_config_for_update(store.clone())
|
||||
.await
|
||||
.expect("published tier config fixture should load with metadata");
|
||||
let current_etag = current_etag.expect("published tier config fixture should have an ETag");
|
||||
let affected_targets = build_tier_mutation_affected_targets(
|
||||
TierMutationIntentKind::Add,
|
||||
HashSet::from(["COLD-A".to_string()]),
|
||||
&empty_mgr(),
|
||||
&persisted,
|
||||
)
|
||||
.expect("published AddTier targets should build");
|
||||
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &persisted, affected_targets)
|
||||
.expect("published AddTier intent should build")
|
||||
.expect("published AddTier should require a durable intent");
|
||||
intent
|
||||
.advance(TierMutationIntentState::Committed, Some(current_etag))
|
||||
.expect("published AddTier intent should commit");
|
||||
save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent)
|
||||
.await
|
||||
.expect("published coordinator intent should persist");
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("published peer intent should persist");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("published"));
|
||||
}
|
||||
{
|
||||
let guard = manager.read().await;
|
||||
assert_eq!(
|
||||
tier_config_candidate_digest(&guard).expect("published manager digest should build"),
|
||||
intent.candidate_digest
|
||||
);
|
||||
}
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&manager, &intent)
|
||||
.await
|
||||
.expect("pre-existing committed runtime fence should install");
|
||||
assert!(
|
||||
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await.is_err(),
|
||||
"fixture must begin with the committed runtime fence installed"
|
||||
);
|
||||
|
||||
let started = Arc::new(Notify::new());
|
||||
let release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(
|
||||
vec![Arc::new(BlockingCommitTierMutationPeer {
|
||||
started: started.clone(),
|
||||
release: release.clone(),
|
||||
})],
|
||||
async {
|
||||
let reload = TierConfigMgr::reload_handle_with(&manager, store.clone());
|
||||
tokio::pin!(reload);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::select! {
|
||||
result = &mut reload => panic!("reload finished before terminal replay was released: {result:?}"),
|
||||
_ = started.notified() => {}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("terminal replay should reach the blocking peer");
|
||||
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("terminal cleanup must not re-fence an already-published tier");
|
||||
drop(lease);
|
||||
release.add_permits(1);
|
||||
tokio::time::timeout(Duration::from_secs(5), &mut reload)
|
||||
.await
|
||||
.expect("terminal replay should finish after the peer responds")
|
||||
.expect("terminal replay should succeed after the peer responds");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(
|
||||
TierConfigMgr::load_coordinator_mutation_intents(store.clone())
|
||||
.await
|
||||
.expect("coordinator cleanup should be readable")
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
TierConfigMgr::load_tier_mutation_intents(store)
|
||||
.await
|
||||
.expect("retained peer tombstone should be readable"),
|
||||
vec![intent]
|
||||
);
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
assert!(
|
||||
lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(),
|
||||
"retained terminal evidence must not restore the published runtime fence"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_terminal_tombstone_gc_uses_etag_and_retains_racing_replacement() {
|
||||
use crate::services::tier::tier_mutation_intent::{
|
||||
@@ -11716,9 +11597,8 @@ mod tests {
|
||||
assert!(merged[0].has_peer_record && merged[0].has_coordinator_record);
|
||||
}
|
||||
|
||||
let err =
|
||||
TierConfigMgr::merge_mutation_recovery_intents(std::slice::from_ref(&committed), std::slice::from_ref(&prepared))
|
||||
.expect_err("a peer committed record cannot outrun the coordinator commit order");
|
||||
let err = TierConfigMgr::merge_mutation_recovery_intents(&[committed.clone()], &[prepared.clone()])
|
||||
.expect_err("a peer committed record cannot outrun the coordinator commit order");
|
||||
assert!(err.to_string().contains("conflicting states"), "{err}");
|
||||
|
||||
let mut conflicting_identity = prepared.clone();
|
||||
@@ -13563,11 +13443,9 @@ mod tests {
|
||||
let build = tokio::spawn(async move { TierConfigMgr::acquire_operation_lease(&build_manager, cold_tier).await });
|
||||
barrier.arrived.notified().await;
|
||||
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("cold driver construction must not block manager readers"),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("cold driver construction must not block manager readers");
|
||||
let tier_b = tokio::time::timeout(Duration::from_millis(100), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
|
||||
.await
|
||||
.expect("cold tier A construction must not block tier B")
|
||||
@@ -13770,11 +13648,9 @@ mod tests {
|
||||
let verify_manager = manager.clone();
|
||||
let verify = tokio::spawn(async move { TierConfigMgr::verify_without_manager_lock(&verify_manager, "COLD-A").await });
|
||||
started.notified().await;
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("slow verify must not hold the manager lock"),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.read())
|
||||
.await
|
||||
.expect("slow verify must not hold the manager lock");
|
||||
release.add_permits(1);
|
||||
verify.await.expect("verify task should join").expect("verify should finish");
|
||||
}
|
||||
@@ -14182,11 +14058,9 @@ mod tests {
|
||||
vec!["COLD-A".to_string()]
|
||||
);
|
||||
}
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(1), manager.read())
|
||||
.await
|
||||
.expect("manager reads must not wait for tier A leases"),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(1), manager.read())
|
||||
.await
|
||||
.expect("manager reads must not wait for tier A leases");
|
||||
let next_b = tokio::time::timeout(Duration::from_secs(1), TierConfigMgr::acquire_operation_lease(&manager, "COLD-B"))
|
||||
.await
|
||||
.expect("tier B lease acquisition must not wait for tier A")
|
||||
@@ -14619,7 +14493,7 @@ mod tests {
|
||||
"https://example-compat.invalid"
|
||||
);
|
||||
let runtime = registered_tier_driver_runtime(&manager_guard).expect("runtime sidecar should remain registered");
|
||||
assert!(!lock_unpoisoned(&runtime).generations.contains_key("COLD-A"));
|
||||
assert!(lock_unpoisoned(&runtime).generations.get("COLD-A").is_none());
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -15259,12 +15133,15 @@ mod tests {
|
||||
.filter(|object| object.bucket == bucket && object.name.starts_with(prefix))
|
||||
.cloned()
|
||||
.collect();
|
||||
objects.sort_by_key(tier_test_object_marker);
|
||||
objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
|
||||
if marker.is_some() || version_marker.is_some() {
|
||||
let marker = (marker.unwrap_or_default(), version_marker.unwrap_or_default());
|
||||
objects.retain(|object| tier_test_object_marker(object) > marker);
|
||||
}
|
||||
let limit: usize = usize::try_from(max_keys).unwrap_or_default();
|
||||
let limit = match usize::try_from(max_keys) {
|
||||
Ok(limit) => limit,
|
||||
Err(_) => 0,
|
||||
};
|
||||
let is_truncated = objects.len() > limit;
|
||||
if is_truncated {
|
||||
objects.truncate(limit);
|
||||
@@ -15294,16 +15171,17 @@ mod tests {
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<()> {
|
||||
if self.fail_reference_walk.load(Ordering::SeqCst)
|
||||
&& result
|
||||
if self.fail_reference_walk.load(Ordering::SeqCst) {
|
||||
if result
|
||||
.send(StorageObjectInfoOrErr {
|
||||
item: None,
|
||||
err: Some(Error::other("injected tier reference walk failure")),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Ok(());
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let mut objects = self
|
||||
.listed_versions
|
||||
@@ -15314,7 +15192,7 @@ mod tests {
|
||||
.filter(|object| opts.include_free_versions || !object.transitioned_object.free_version)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
objects.sort_by_key(tier_test_object_marker);
|
||||
objects.sort_by(|left, right| tier_test_object_marker(left).cmp(&tier_test_object_marker(right)));
|
||||
if let Some(marker) = opts.marker.as_deref() {
|
||||
objects.retain(|object| object.name.as_str() > marker);
|
||||
}
|
||||
@@ -15492,18 +15370,17 @@ mod tests {
|
||||
api_view.rustfs.expect("admin RustFS payload should exist").secret_key,
|
||||
TIER_CREDENTIAL_REDACTED
|
||||
);
|
||||
{
|
||||
let observed = lock_unpoisoned(&observed);
|
||||
assert_eq!(observed.len(), 1);
|
||||
assert_eq!(
|
||||
observed[0]
|
||||
.rustfs
|
||||
.as_ref()
|
||||
.expect("backend factory should observe the RustFS payload")
|
||||
.secret_key,
|
||||
SECRET_KEY
|
||||
);
|
||||
}
|
||||
let observed = lock_unpoisoned(&observed);
|
||||
assert_eq!(observed.len(), 1);
|
||||
assert_eq!(
|
||||
observed[0]
|
||||
.rustfs
|
||||
.as_ref()
|
||||
.expect("backend factory should observe the RustFS payload")
|
||||
.secret_key,
|
||||
SECRET_KEY
|
||||
);
|
||||
drop(observed);
|
||||
|
||||
let operations = backend.op_log().await;
|
||||
assert_eq!(operations.len(), 5);
|
||||
@@ -16704,7 +16581,7 @@ mod tests {
|
||||
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
candidate.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
|
||||
|
||||
let targets = TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true)
|
||||
let targets = TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true)
|
||||
.affected_targets(¤t, &candidate)
|
||||
.expect("add proof should ignore unchanged durable tiers");
|
||||
assert_eq!(targets.len(), 1);
|
||||
@@ -16730,7 +16607,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store.clone(),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -16789,15 +16666,14 @@ mod tests {
|
||||
.await
|
||||
.expect("legacy nested-name Add must run the full coordinator fanout");
|
||||
|
||||
{
|
||||
let prepared_intents = lock_unpoisoned(&prepared_intents);
|
||||
assert_eq!(prepared_intents.len(), 1);
|
||||
assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add);
|
||||
assert_eq!(prepared_intents[0].affected_targets.len(), 1);
|
||||
assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY");
|
||||
assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none());
|
||||
assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some());
|
||||
}
|
||||
let prepared_intents = lock_unpoisoned(&prepared_intents);
|
||||
assert_eq!(prepared_intents.len(), 1);
|
||||
assert_eq!(prepared_intents[0].kind, TierMutationIntentKind::Add);
|
||||
assert_eq!(prepared_intents[0].affected_targets.len(), 1);
|
||||
assert_eq!(prepared_intents[0].affected_targets[0].tier_name, "COLD-LEGACY");
|
||||
assert!(prepared_intents[0].affected_targets[0].old_backend_identity.is_none());
|
||||
assert!(prepared_intents[0].affected_targets[0].new_backend_identity.is_some());
|
||||
drop(prepared_intents);
|
||||
|
||||
let peer_calls = lock_unpoisoned(&peer_calls).clone();
|
||||
let prepare_index = peer_calls
|
||||
@@ -16879,7 +16755,7 @@ mod tests {
|
||||
let err = TierConfigMgr::update_candidate_with_config_lock(
|
||||
&manager,
|
||||
store.clone(),
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-B")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-B"), true),
|
||||
)
|
||||
.await
|
||||
.expect_err("a new tier config update must wait for pending mutation recovery");
|
||||
@@ -17155,7 +17031,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -17206,7 +17082,7 @@ mod tests {
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -17269,7 +17145,7 @@ mod tests {
|
||||
TierConfigMgr::prevalidate_candidate_owned(
|
||||
empty_mgr(),
|
||||
None,
|
||||
TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true),
|
||||
TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -17933,7 +17809,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(Box::new(build_rustfs_tier("COLD-A")), true)).await;
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
#[cfg(feature = "gcs")]
|
||||
@@ -717,7 +719,17 @@ async fn check_warm_backend_with_deadlines(
|
||||
if !matches!(cleanup_result, Ok(Ok(()))) {
|
||||
return Err(probe_cleanup_incomplete_error());
|
||||
}
|
||||
read_result?;
|
||||
if let Err(err) = read_result {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
/*else if is_err_signature_does_not_match(err) {
|
||||
return Err(ERR_TIER_MISSING_CREDENTIALS);
|
||||
}*/
|
||||
//else {
|
||||
return Err(err);
|
||||
//}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -747,7 +759,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -788,7 +800,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -808,7 +820,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -828,7 +840,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -848,7 +860,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -868,7 +880,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -888,7 +900,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -917,7 +929,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
@@ -937,7 +949,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -104,35 +106,39 @@ impl WarmBackendS3 {
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let has_web_identity_token_file = !conf.aws_role_web_identity_token_file.is_empty();
|
||||
let has_role_arn = !conf.aws_role_arn.is_empty();
|
||||
let has_access_key = !conf.access_key.is_empty();
|
||||
let has_secret_key = !conf.secret_key.is_empty();
|
||||
|
||||
if has_web_identity_token_file != has_role_arn {
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
{
|
||||
return Err(std::io::Error::other("both the token file and the role ARN are required"));
|
||||
} else if has_access_key != has_secret_key {
|
||||
} else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
|
||||
return Err(std::io::Error::other("both the access and secret keys are required"));
|
||||
} else if conf.aws_role && (has_web_identity_token_file || has_role_arn || has_access_key || has_secret_key) {
|
||||
} else if conf.aws_role
|
||||
&& (conf.aws_role_web_identity_token_file != ""
|
||||
|| conf.aws_role_arn != ""
|
||||
|| conf.access_key != ""
|
||||
|| conf.secret_key != "")
|
||||
{
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket.is_empty() {
|
||||
} else if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
let creds = if has_access_key && has_secret_key {
|
||||
let creds: Credentials<Static>;
|
||||
|
||||
if conf.access_key != "" && conf.secret_key != "" {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
Credentials::new(Static(Value {
|
||||
creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
secret_access_key: conf.secret_key.clone(),
|
||||
session_token: "".to_string(),
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}))
|
||||
}));
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
};
|
||||
}
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let opts = Options {
|
||||
creds,
|
||||
@@ -156,11 +162,11 @@ impl WarmBackendS3 {
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix, object)
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_with_result(&self, object: &str, rv: &str) -> Result<RemoveObjectResult, std::io::Error> {
|
||||
@@ -407,10 +413,6 @@ impl TransitionCandidateVersions {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
clippy::items_after_test_module,
|
||||
reason = "keep parsing tests adjacent to the helpers they cover"
|
||||
)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
@@ -915,7 +917,7 @@ impl WarmBackend for WarmBackendS3 {
|
||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
||||
.await?;
|
||||
|
||||
Ok(!result.common_prefixes.is_empty() || !result.contents.is_empty())
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -883,17 +883,6 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionKillPoint as SetDiskTransitionTransactionKillPoint,
|
||||
TransitionTransactionKillPointBarrier as SetDiskTransitionTransactionKillPointBarrier,
|
||||
};
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::{
|
||||
TransitionTransactionMutationKind as SetDiskTransitionTransactionMutationKind,
|
||||
TransitionTransactionMutationObservation as SetDiskTransitionTransactionMutationObservation,
|
||||
TransitionTransactionMutationProbe as SetDiskTransitionTransactionMutationProbe,
|
||||
};
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
@@ -6463,114 +6452,113 @@ pub fn should_heal_object_on_disk(
|
||||
(false, false, None)
|
||||
}
|
||||
|
||||
/// Probe every drive of the set at once. Each live probe is bounded by the
|
||||
/// drive `disk_info` timeout, and the admin peer probe budget only covers one
|
||||
/// such timeout; a sequential walk over several stalled drives after a power
|
||||
/// cut would exceed it and make healthy peers render as unknown (#6488).
|
||||
async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<rustfs_madmin::Disk> {
|
||||
join_all(disks.iter().zip(eps).map(|(disk, ep)| disk_admin_info(disk.as_ref(), ep))).await
|
||||
}
|
||||
let mut ret = Vec::new();
|
||||
|
||||
async fn disk_admin_info(disk: Option<&DiskStore>, ep: &Endpoint) -> rustfs_madmin::Disk {
|
||||
let Some(disk) = disk else {
|
||||
return rustfs_madmin::Disk {
|
||||
endpoint: ep.to_string(),
|
||||
drive_path: ep.get_file_path(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
runtime_state: None,
|
||||
offline_duration_seconds: None,
|
||||
state: DiskError::DiskNotFound.to_string(),
|
||||
capacity_observation_source: Some("missing".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
for (i, pool) in disks.iter().enumerate() {
|
||||
if let Some(disk) = pool {
|
||||
let runtime_state = disk.runtime_state();
|
||||
let offline_duration_seconds = disk.offline_duration_secs();
|
||||
let capacity_snapshot = disk.last_capacity_snapshot();
|
||||
let cached_disk_id = disk.cached_disk_id().await;
|
||||
if runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect {
|
||||
match disk
|
||||
.disk_info(&DiskInfoOptions {
|
||||
metrics: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
disk.record_capacity_probe(res.total, res.used, res.free);
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
endpoint: eps[i].to_string(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
state: "ok".to_owned(),
|
||||
|
||||
let runtime_state = disk.runtime_state();
|
||||
let offline_duration_seconds = disk.offline_duration_secs();
|
||||
let capacity_snapshot = disk.last_capacity_snapshot();
|
||||
let cached_disk_id = disk.cached_disk_id().await;
|
||||
if !(runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect) {
|
||||
let mut disk_info = build_runtime_snapshot_disk(ep, runtime_state, offline_duration_seconds, capacity_snapshot);
|
||||
disk_info.metrics = disk.metrics_snapshot();
|
||||
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
|
||||
return disk_info;
|
||||
}
|
||||
root_disk: res.root_disk,
|
||||
drive_path: res.mount_path.clone(),
|
||||
healing: res.healing,
|
||||
scanning: res.scanning,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
capacity_observation_source: Some("live_probe".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
|
||||
match disk
|
||||
.disk_info(&DiskInfoOptions {
|
||||
metrics: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
disk.record_capacity_probe(res.total, res.used, res.free);
|
||||
rustfs_madmin::Disk {
|
||||
endpoint: ep.to_string(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
state: "ok".to_owned(),
|
||||
|
||||
root_disk: res.root_disk,
|
||||
drive_path: res.mount_path.clone(),
|
||||
healing: res.healing,
|
||||
scanning: res.scanning,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
capacity_observation_source: Some("live_probe".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
|
||||
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
|
||||
major: res.major as u32,
|
||||
minor: res.minor as u32,
|
||||
model: None,
|
||||
total_space: res.total,
|
||||
used_space: res.used,
|
||||
available_space: res.free,
|
||||
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
|
||||
utilization: utilization_percent(res.total, res.used),
|
||||
used_inodes: res.used_inodes,
|
||||
free_inodes: res.free_inodes,
|
||||
metrics: Some(res.metrics),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let mut disk_info = rustfs_madmin::Disk {
|
||||
state: err.to_string(),
|
||||
endpoint: ep.to_string(),
|
||||
drive_path: ep.get_file_path(),
|
||||
local: ep.is_local,
|
||||
pool_index: ep.pool_idx,
|
||||
set_index: ep.set_idx,
|
||||
disk_index: ep.disk_idx,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
metrics: disk.metrics_snapshot(),
|
||||
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some((total, used, free, _)) = capacity_snapshot {
|
||||
disk_info.total_space = total;
|
||||
disk_info.used_space = used;
|
||||
disk_info.available_space = free;
|
||||
disk_info.utilization = utilization_percent(total, used);
|
||||
disk_info.capacity_observation_source = Some("snapshot".to_owned());
|
||||
disk_info.capacity_observation_age_seconds =
|
||||
capacity_snapshot.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
|
||||
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
|
||||
major: res.major as u32,
|
||||
minor: res.minor as u32,
|
||||
model: None,
|
||||
total_space: res.total,
|
||||
used_space: res.used,
|
||||
available_space: res.free,
|
||||
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
|
||||
utilization: utilization_percent(res.total, res.used),
|
||||
used_inodes: res.used_inodes,
|
||||
free_inodes: res.free_inodes,
|
||||
metrics: Some(res.metrics),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
let mut disk_info = rustfs_madmin::Disk {
|
||||
state: err.to_string(),
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
metrics: disk.metrics_snapshot(),
|
||||
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some((total, used, free, _)) = capacity_snapshot {
|
||||
disk_info.total_space = total;
|
||||
disk_info.used_space = used;
|
||||
disk_info.available_space = free;
|
||||
disk_info.utilization = utilization_percent(total, used);
|
||||
disk_info.capacity_observation_source = Some("snapshot".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = capacity_snapshot
|
||||
.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
|
||||
} else {
|
||||
disk_info.capacity_observation_source = Some("missing".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = Some(0);
|
||||
}
|
||||
ret.push(disk_info);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
disk_info.capacity_observation_source = Some("missing".to_owned());
|
||||
disk_info.capacity_observation_age_seconds = Some(0);
|
||||
let mut disk_info =
|
||||
build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds, capacity_snapshot);
|
||||
disk_info.metrics = disk.metrics_snapshot();
|
||||
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
|
||||
ret.push(disk_info);
|
||||
}
|
||||
disk_info
|
||||
} else {
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
runtime_state: None,
|
||||
offline_duration_seconds: None,
|
||||
state: DiskError::DiskNotFound.to_string(),
|
||||
capacity_observation_source: Some("missing".to_owned()),
|
||||
capacity_observation_age_seconds: Some(0),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
fn build_runtime_snapshot_disk(
|
||||
@@ -10692,41 +10680,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_get_disks_info_probes_drives_concurrently() {
|
||||
use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
|
||||
|
||||
let format = FormatV3::new(1, 4);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_idx in 0..4 {
|
||||
let (dir, endpoint, disk) = make_formatted_local_disk_for_info_test(disk_idx, &format).await;
|
||||
temp_dirs.push(dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let probe_delay = std::time::Duration::from_secs(2);
|
||||
let started = tokio::time::Instant::now();
|
||||
let info = DISK_INFO_PROBE_DELAY_FOR_TEST
|
||||
.scope(probe_delay, get_disks_info(&disks, &endpoints))
|
||||
.await;
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
assert_eq!(info.len(), 4);
|
||||
assert!(info.iter().all(|disk| disk.state == "ok"), "every drive should still report a live probe");
|
||||
assert_eq!(
|
||||
info.iter().map(|disk| disk.disk_index).collect::<Vec<_>>(),
|
||||
endpoints.iter().map(|ep| ep.disk_idx).collect::<Vec<_>>(),
|
||||
"concurrent probes must keep endpoint order"
|
||||
);
|
||||
assert!(
|
||||
elapsed < probe_delay * 2,
|
||||
"four stalled drives must cost one probe delay, not four; took {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_disks_info_preserves_remote_cached_disk_id_when_offline() {
|
||||
let (endpoint, disk) = make_remote_disk_for_info_test(0).await;
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::super::{
|
||||
};
|
||||
use crate::disk::DataDirDeleteStatus;
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::disk::local::{DELETE_DATA_DIR_MARKER_PREFIX, metadata_less_part_file};
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
@@ -301,6 +301,12 @@ struct MetadataLessDataDirCleanup {
|
||||
touched_disks: Vec<bool>,
|
||||
}
|
||||
|
||||
fn metadata_less_part_file(entry: &str) -> bool {
|
||||
entry
|
||||
.strip_prefix("part.")
|
||||
.is_some_and(|part_number| part_number.parse::<usize>().is_ok_and(|part_number| part_number > 0))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DanglingCheckPartsFailure {
|
||||
key: DanglingCheckPartsFailureKey,
|
||||
|
||||
@@ -3301,18 +3301,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
||||
Self::assign_rename_data_indexes(&mut parts_metadatas);
|
||||
// Disk deadlines can expire before physical publication or failure undo drains.
|
||||
let mut rename_result = SetDisks::rename_data_owned_with_fence(
|
||||
let mut rename_result = SetDisks::rename_data_owned(
|
||||
&commit_disks,
|
||||
(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path),
|
||||
parts_metadatas,
|
||||
(&commit_bucket, &commit_object),
|
||||
write_quorum,
|
||||
commit_allows_early_ack,
|
||||
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(write_quorum, None)
|
||||
.with_namespace_commit_guard(
|
||||
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
if let Ok(rename_commit) = rename_result.as_mut() {
|
||||
@@ -6763,301 +6758,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_advances_namespace_generation_after_commit() {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-namespace-commit";
|
||||
let object = "completed-object";
|
||||
let body = vec![0x65; 4096];
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let before = set_disks.ctx.namespace_commit_generation();
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before, "staging is not publication");
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("completion must finish")
|
||||
.expect("the four real shards must commit");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET after completion must finish")
|
||||
.expect("a successful completion must be immediately readable");
|
||||
let mut actual = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("the completed body stream must finish")
|
||||
.expect("read completed body");
|
||||
assert_eq!(actual, body);
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists());
|
||||
}
|
||||
assert!(matches!(
|
||||
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
assert_eq!(
|
||||
set_disks.ctx.namespace_commit_generation(),
|
||||
before + 2,
|
||||
"one completed MPU must invalidate snapshots at namespace admission and physical retirement"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn assert_complete_multipart_physical_namespace_owner(undo: bool) {
|
||||
use crate::disk::os::{self, prepared_publication_test_hooks as hooks};
|
||||
use crate::set_disk::core::io_primitives::rename_fault_injection;
|
||||
use futures::FutureExt;
|
||||
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let (dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-physical-namespace";
|
||||
let object = if undo { "undo-tail" } else { "publication-tail" };
|
||||
let old_body = vec![0x41; 1024];
|
||||
let new_body = vec![0x62; 4096];
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let old = set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(old_body.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed a real readable old version");
|
||||
let old_etag = old.etag.expect("the committed old object must have an ETag");
|
||||
let before = set_disks.ctx.namespace_commit_generation();
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &new_body, &ObjectOptions::default()).await;
|
||||
let new_etag = get_complete_multipart_md5(&parts);
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for dir in &dirs {
|
||||
assert!(dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists());
|
||||
}
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before);
|
||||
let _fault = undo.then(|| rename_fault_injection::fail_rename_on(object, &[2, 3]));
|
||||
let stage = if undo {
|
||||
hooks::Stage::Rename
|
||||
} else {
|
||||
hooks::Stage::PreparedRename
|
||||
};
|
||||
let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let mut hooks = Vec::new();
|
||||
let mut releases = Vec::new();
|
||||
let mut mutation_paths = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical MPU fixture requires local disks");
|
||||
};
|
||||
let destination = local
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, object)
|
||||
.expect("leased IO path");
|
||||
let entered_tx = entered_tx.clone();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(hooks::install_at(stage, &destination.join(STORAGE_FORMAT_FILE), move || {
|
||||
let _ = entered_tx.send(index);
|
||||
// Dropping senders releases every syscall on assertion failure, too.
|
||||
let _ = release_rx.recv();
|
||||
}));
|
||||
releases.push(release_tx);
|
||||
// Canonical rename serializes the object directory; backup restore
|
||||
// serializes its xl.meta destination. Drain the actual executor key.
|
||||
mutation_paths.push(if undo {
|
||||
destination.join(STORAGE_FORMAT_FILE)
|
||||
} else {
|
||||
destination
|
||||
});
|
||||
}
|
||||
drop(entered_tx);
|
||||
let complete_set = set_disks.clone();
|
||||
let complete_upload = upload_id.clone();
|
||||
let mut complete = tokio::spawn(async move {
|
||||
complete_set
|
||||
.complete_multipart_upload(bucket, object, &complete_upload, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
let mut complete_joined = false;
|
||||
let mut observed_counts = None;
|
||||
let observations = std::panic::AssertUnwindSafe(async {
|
||||
let expected_publishers = if undo { 2 } else { 4 };
|
||||
let entered = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let mut entered = HashSet::new();
|
||||
while entered.len() < expected_publishers {
|
||||
tokio::select! {
|
||||
index = entered_rx.recv() => {
|
||||
assert!(entered.insert(index.expect("physical publisher must signal entry")));
|
||||
}
|
||||
result = &mut complete => {
|
||||
complete_joined = true;
|
||||
panic!("completion returned before physical entry: {result:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
entered
|
||||
})
|
||||
.await
|
||||
.expect("all expected physical metadata operations must enter");
|
||||
let pending_at_entry = set_disks.ctx.namespace_commits_pending();
|
||||
let generation_at_entry = set_disks.ctx.namespace_commit_generation();
|
||||
for &index in &entered {
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
disks[index].read_version("", bucket, object, "", &ReadOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("metadata observation must finish while publication is paused")
|
||||
.expect("metadata before the paused physical action must be readable");
|
||||
assert_eq!(
|
||||
metadata.metadata.get("etag"),
|
||||
Some(if undo { &new_etag } else { &old_etag }),
|
||||
"undo must follow actual publication; prepared rename must precede publication"
|
||||
);
|
||||
}
|
||||
|
||||
// The entry signals run inside the real blocking closures, after each
|
||||
// wrapper installed its normal deadline. No quota/external guard disables it.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let joined = tokio::time::timeout(Duration::from_secs(5), &mut complete).await;
|
||||
complete_joined = joined.is_ok();
|
||||
let result = joined
|
||||
.expect("ordinary MPU disk/undo deadlines must still return before physical drain")
|
||||
.expect("completion task must not panic");
|
||||
assert!(result.is_err(), "a timed-out or two-shard commit cannot acknowledge success");
|
||||
let pending_after_timeout = set_disks.ctx.namespace_commits_pending();
|
||||
let generation_after_timeout = set_disks.ctx.namespace_commit_generation();
|
||||
observed_counts = Some((pending_at_entry, generation_at_entry, pending_after_timeout, generation_after_timeout));
|
||||
for &index in &entered {
|
||||
assert!(
|
||||
os::acquire_rename_data_mutation_lease(&disks[index].path(), bucket, &mutation_paths[index])
|
||||
.now_or_never()
|
||||
.is_none(),
|
||||
"timed-out physical work must still own its object serialization"
|
||||
);
|
||||
}
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path).exists(),
|
||||
"failed completion must not clean the upload staging"
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
|
||||
// Release even after a failed observation, then finish dispatch before
|
||||
// draining every physical key. No per-disk assertion may skip a later drain.
|
||||
drop(releases);
|
||||
drop(hooks);
|
||||
let coordinator_drained = complete_joined
|
||||
|| tokio::time::timeout(Duration::from_secs(10), &mut complete).await.is_ok();
|
||||
if !coordinator_drained {
|
||||
complete.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), &mut complete).await;
|
||||
}
|
||||
let drains = futures::future::join_all(disks.iter().zip(&mutation_paths).map(|(disk, destination)| async move {
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, destination),
|
||||
)
|
||||
.await
|
||||
.map(drop)
|
||||
}))
|
||||
.await;
|
||||
let owner_drained = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while set_disks.ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let physical_drained = drains.iter().all(|drain| drain.is_ok());
|
||||
if !coordinator_drained || !physical_drained || owner_drained.is_err() {
|
||||
// A bounded cleanup failure cannot justify deleting roots that a
|
||||
// detached executor might still use. Keep them for diagnosis.
|
||||
let retained = dirs.into_iter().map(TempDir::keep).collect::<Vec<_>>();
|
||||
eprintln!("MPU cleanup incomplete: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
panic!("MPU cleanup did not drain: coordinator={coordinator_drained}, physical={physical_drained}, retained={retained:?}");
|
||||
}
|
||||
if let Err(panic) = observations {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// Preserve the original drain checks after collecting every result.
|
||||
for drained in drains {
|
||||
drained.expect("released physical MPU work must drain");
|
||||
}
|
||||
owner_drained.expect("physical retirement must finish its namespace counter decrement");
|
||||
for disk in &disks {
|
||||
let metadata = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("all disks must expose the expected final metadata");
|
||||
assert_eq!(metadata.metadata.get("etag"), Some(if undo { &old_etag } else { &new_etag }));
|
||||
}
|
||||
let (pending_at_entry, generation_at_entry, pending_after_timeout, generation_after_timeout) =
|
||||
observed_counts.expect("successful observations must record namespace counters");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET after physical drain must finish")
|
||||
.expect("the real final object must be readable");
|
||||
let mut actual = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual))
|
||||
.await
|
||||
.expect("the final object stream must finish")
|
||||
.expect("read final object bytes");
|
||||
assert_eq!(actual, if undo { old_body } else { new_body });
|
||||
assert!(
|
||||
pending_at_entry && pending_after_timeout,
|
||||
"physical MPU work outlived namespace accounting: undo={undo}"
|
||||
);
|
||||
assert_eq!(generation_at_entry, before + 1);
|
||||
assert_eq!(
|
||||
generation_after_timeout, generation_at_entry,
|
||||
"the blocked physical owner cannot retire early"
|
||||
);
|
||||
assert_eq!(set_disks.ctx.namespace_commit_generation(), before + 2);
|
||||
assert!(!set_disks.ctx.namespace_commits_pending());
|
||||
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_timeout_keeps_namespace_owner_until_physical_publication() {
|
||||
assert_complete_multipart_physical_namespace_owner(false).await;
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_failed_quorum_keeps_namespace_owner_until_physical_undo() {
|
||||
assert_complete_multipart_physical_namespace_owner(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -630,14 +630,27 @@ impl ECStore {
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
let (meta, pool_meta_replica_state) = {
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?;
|
||||
load_pool_meta_for_startup(self.pools.clone(), &mut write_state).await?
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
load.await?
|
||||
};
|
||||
let update = meta.validate(self.pools.clone())?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": pool_meta_replica_state.needs_repair,
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
@@ -649,15 +662,17 @@ impl ECStore {
|
||||
// distributed startup can race on the same lock and replay the prior init bug.
|
||||
{
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
installed_pool_meta = persist_pool_meta_for_startup_if_safe(
|
||||
let persist = persist_pool_meta_for_startup_if_safe(
|
||||
&installed_pool_meta,
|
||||
self.pools.clone(),
|
||||
pool_meta_replica_state,
|
||||
&mut write_state,
|
||||
update,
|
||||
should_persist_pool_meta,
|
||||
)
|
||||
.await?;
|
||||
);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
|
||||
installed_pool_meta = persist.await?;
|
||||
}
|
||||
|
||||
{
|
||||
@@ -828,17 +843,7 @@ mod tests {
|
||||
recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
|
||||
IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
|
||||
observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent,
|
||||
},
|
||||
recovery_disposition::{
|
||||
IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage,
|
||||
dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once,
|
||||
load_recovery_disposition,
|
||||
},
|
||||
recovery_disposition_runtime::garbage_collect_completed_recovery_disposition,
|
||||
recovery_export::{
|
||||
create_recovery_export, inspect_recovery_export_observation, load_recovery_export,
|
||||
recovery_export_record_object_name,
|
||||
observe_recovery_source, save_recovery_control_if_absent,
|
||||
},
|
||||
tier_delete_journal::{
|
||||
DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
|
||||
@@ -862,10 +867,9 @@ mod tests {
|
||||
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, recover_transition_transaction_records_at,
|
||||
retry_transition_recovery_for_operator, save_transition_transaction_record,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
load_transition_transaction_record, recover_transition_transaction_records,
|
||||
recover_transition_transaction_records_at, save_transition_transaction_record,
|
||||
save_transition_transaction_record_if_current, transition_recovery_control_id,
|
||||
transition_transaction_record_object_name,
|
||||
},
|
||||
@@ -877,11 +881,7 @@ mod tests {
|
||||
data_movement::SourceCleanupDeleteBarrier,
|
||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE},
|
||||
runtime::{global::set_object_store_resolver, sources as runtime_sources},
|
||||
services::notification_sys::{
|
||||
acquire_tier_delete_journal_fleet_proof, acquire_transition_transaction_compaction_fleet_proof,
|
||||
install_transition_transaction_compaction_fleet_proof_for_test,
|
||||
transition_transaction_compaction_fleet_proof_matches,
|
||||
},
|
||||
services::notification_sys::acquire_tier_delete_journal_fleet_proof,
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{
|
||||
@@ -903,14 +903,7 @@ mod tests {
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend},
|
||||
},
|
||||
set_disk::{
|
||||
SetDiskTransitionTransactionKillPoint as TransitionTransactionKillPoint,
|
||||
SetDiskTransitionTransactionKillPointBarrier as TransitionTransactionKillPointBarrier,
|
||||
SetDiskTransitionTransactionMutationKind as TransitionTransactionMutationKind,
|
||||
SetDiskTransitionTransactionMutationObservation as TransitionTransactionMutationObservation,
|
||||
SetDiskTransitionTransactionMutationProbe as TransitionTransactionMutationProbe,
|
||||
SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
},
|
||||
set_disk::SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
storage_api_contracts::list::ListOperations as _,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -4391,7 +4384,7 @@ mod tests {
|
||||
}
|
||||
retry_source_info.parts = Arc::new(retry_source_parts);
|
||||
assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
|
||||
assert!(retry_source_info.is_multipart());
|
||||
assert!(!retry_source_info.is_multipart());
|
||||
assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
|
||||
assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
|
||||
assert!(
|
||||
@@ -11597,36 +11590,6 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn only_transition_transaction(store: Arc<crate::store::ECStore>) -> TransitionTransaction {
|
||||
let records = store
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX,
|
||||
None,
|
||||
None,
|
||||
100,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("transition transaction records should be listable")
|
||||
.objects;
|
||||
assert_eq!(records.len(), 1, "test fixture should have exactly one transition transaction");
|
||||
let transaction_id = records[0]
|
||||
.name
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.and_then(|name| name.strip_suffix(".json"))
|
||||
.and_then(|name| uuid::Uuid::parse_str(name).ok())
|
||||
.expect("transition transaction path should end in its UUID");
|
||||
load_transition_transaction_record(store, transaction_id)
|
||||
.await
|
||||
.expect("transition transaction should load")
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_transition_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
@@ -16922,43 +16885,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned);
|
||||
let mut created_exports = Vec::new();
|
||||
for exportable in first_controls
|
||||
.iter()
|
||||
.filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
|
||||
{
|
||||
let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id)
|
||||
.await
|
||||
.expect("fresh legacy recovery observation should be exportable");
|
||||
let created = create_recovery_export(store.clone(), &observation, &creator_sha256)
|
||||
.await
|
||||
.expect("legacy recovery export should be created exactly once");
|
||||
assert!(!created.replayed);
|
||||
let loaded = load_recovery_export(store.clone(), &created.export_id)
|
||||
.await
|
||||
.expect("created legacy recovery export should load");
|
||||
assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes");
|
||||
let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256)
|
||||
.await
|
||||
.expect("the same observed generation should replay its immutable export");
|
||||
assert!(replayed.replayed);
|
||||
assert_eq!(replayed.encoded, created.encoded);
|
||||
created_exports.push(created);
|
||||
}
|
||||
assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path");
|
||||
|
||||
let corrupt_export_id = &created_exports[0].export_id;
|
||||
let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id)
|
||||
.expect("export path should build");
|
||||
com::save_config(store.clone(), &export_path, Vec::new())
|
||||
.await
|
||||
.expect("zero-byte corruption fixture should persist");
|
||||
let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id)
|
||||
.await
|
||||
.expect_err("an existing zero-byte export must fail closed");
|
||||
assert!(!matches!(corrupt_export, Error::ConfigNotFound));
|
||||
|
||||
com::save_config(
|
||||
store.clone(),
|
||||
&journal_paths[0],
|
||||
@@ -16984,301 +16910,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() {
|
||||
Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await;
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "LEGACY-DISPOSITION";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("legacy disposition tier lease should resolve")
|
||||
.backend_identity();
|
||||
let fixtures = [
|
||||
serde_json::json!({
|
||||
"version": 1,
|
||||
"obj_name": "legacy/disposition-v1",
|
||||
"version_id": "opaque-disposition-v1",
|
||||
"tier_name": tier_name,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": 2,
|
||||
"obj_name": "legacy/disposition-v2",
|
||||
"version_id": "opaque-disposition-v2",
|
||||
"tier_name": tier_name,
|
||||
"backend_identity": backend_identity,
|
||||
}),
|
||||
];
|
||||
let mut journal_paths = Vec::new();
|
||||
for fixture in &fixtures {
|
||||
let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode");
|
||||
let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
|
||||
.expect("legacy disposition fixture should decode");
|
||||
let path = tier_delete_journal_object_name(&entry);
|
||||
com::save_config(store.clone(), &path, data)
|
||||
.await
|
||||
.expect("legacy disposition fixture should persist");
|
||||
journal_paths.push(path);
|
||||
}
|
||||
|
||||
let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("legacy disposition recovery scan should finish");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0));
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
|
||||
|
||||
let mut controls = list_recovery_controls(
|
||||
store.clone(),
|
||||
IlmRecoveryProtocol::TierDeleteJournal,
|
||||
Some(IlmRecoveryClassification::RetainedAmbiguous),
|
||||
100,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("legacy disposition controls should be listable")
|
||||
.records;
|
||||
controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
|
||||
assert_eq!(controls.len(), 2, "both legacy schemas must support disposition");
|
||||
|
||||
let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned);
|
||||
let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned);
|
||||
let wrong_export_sha256 = "ff".repeat(32);
|
||||
for (index, control) in controls.iter().enumerate() {
|
||||
let observation = inspect_recovery_export_observation(store.clone(), &control.control_id)
|
||||
.await
|
||||
.expect("legacy disposition source should be observable");
|
||||
let export = create_recovery_export(store.clone(), &observation, &actor_sha256)
|
||||
.await
|
||||
.expect("legacy disposition export should persist");
|
||||
let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos())
|
||||
.expect("legacy disposition timestamp should fit i64");
|
||||
|
||||
if index == 0 {
|
||||
let wrong_hash = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&wrong_export_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("a mismatched export checksum must fail before local deletion");
|
||||
assert_eq!(wrong_hash, Error::PreconditionFailed);
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 2);
|
||||
}
|
||||
|
||||
let dry_run = dry_run_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
)
|
||||
.await
|
||||
.expect("legacy disposition dry-run should validate exact local state");
|
||||
assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len());
|
||||
assert_eq!(
|
||||
tier_delete_journal_count(store.clone()).await,
|
||||
fixtures.len() - index,
|
||||
"dry-run must not delete a legacy journal"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,)
|
||||
.await,
|
||||
Err(Error::ConfigNotFound)
|
||||
),
|
||||
"dry-run must not persist a disposition record"
|
||||
);
|
||||
|
||||
if index == 0 {
|
||||
inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete);
|
||||
Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("the injected crash must stop after local delete commits");
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
|
||||
let interrupted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
|
||||
.await
|
||||
.expect("the applying disposition must survive the post-delete crash");
|
||||
assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
|
||||
assert!(
|
||||
interrupted.disposition.confirmed_absent.is_empty(),
|
||||
"the crash must occur before absence progress is persisted"
|
||||
);
|
||||
} else {
|
||||
inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon);
|
||||
Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect_err("the injected crash must stop after control abandonment commits");
|
||||
let interrupted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id)
|
||||
.await
|
||||
.expect("the applying disposition must survive the post-control crash");
|
||||
assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying);
|
||||
assert_eq!(
|
||||
interrupted.disposition.confirmed_absent.len(),
|
||||
interrupted.disposition.identity.source_generation.copies.len()
|
||||
);
|
||||
|
||||
let abandoned =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.await
|
||||
.expect("the abandoned control must survive the injected crash");
|
||||
let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode");
|
||||
let mut wrong_history = abandoned.control;
|
||||
wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed;
|
||||
let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id)
|
||||
.expect("control path should remain canonical");
|
||||
com::save_config(
|
||||
store.clone(),
|
||||
&control_path,
|
||||
wrong_history
|
||||
.encode()
|
||||
.expect("the alternate valid control history should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("the alternate control history fixture should persist");
|
||||
let wrong_history_err = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
confirmed_at_unix_nanos + 1,
|
||||
))
|
||||
.await
|
||||
.expect_err("a different abandoned control history must not bridge to completion");
|
||||
assert_eq!(wrong_history_err, Error::PreconditionFailed);
|
||||
com::save_config(store.clone(), &control_path, exact_abandoned)
|
||||
.await
|
||||
.expect("the exact abandoned control fixture should be restored");
|
||||
}
|
||||
|
||||
let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2;
|
||||
let executed = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
replay_confirmed_at_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect("a later request must resume and complete the interrupted disposition");
|
||||
assert_eq!(executed.state, IlmRecoveryDispositionState::Completed);
|
||||
assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed);
|
||||
assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count);
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1);
|
||||
assert!(matches!(
|
||||
com::read_config(store.clone(), &observation.canonical_source_path).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
if index == 0 {
|
||||
let untouched = journal_paths
|
||||
.iter()
|
||||
.find(|path| *path != &observation.canonical_source_path)
|
||||
.expect("the other legacy journal should remain");
|
||||
com::read_config(store.clone(), untouched)
|
||||
.await
|
||||
.expect("disposition must not remove a different legacy journal");
|
||||
}
|
||||
|
||||
let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id)
|
||||
.await
|
||||
.expect("abandoned recovery control should remain inspectable");
|
||||
assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned);
|
||||
assert_eq!(abandoned.control.revision, observation.control_revision + 1);
|
||||
assert_eq!(abandoned.control.observed_source_generation, observation.source_generation);
|
||||
|
||||
let persisted =
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id)
|
||||
.await
|
||||
.expect("completed disposition should remain durable");
|
||||
assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed);
|
||||
|
||||
let replayed = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&actor_sha256,
|
||||
replay_confirmed_at_unix_nanos + 1,
|
||||
))
|
||||
.await
|
||||
.expect("same actor should replay the completed disposition");
|
||||
assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed);
|
||||
assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed);
|
||||
|
||||
let wrong_actor = Box::pin(execute_recovery_disposition(
|
||||
store.clone(),
|
||||
&observation,
|
||||
&export.export_id,
|
||||
&export.content_sha256,
|
||||
&wrong_actor_sha256,
|
||||
replay_confirmed_at_unix_nanos + 2,
|
||||
))
|
||||
.await
|
||||
.expect_err("a different actor must not replay a completed disposition");
|
||||
assert_eq!(wrong_actor, Error::PreconditionFailed);
|
||||
|
||||
assert!(
|
||||
!Box::pin(garbage_collect_completed_recovery_disposition(
|
||||
store.clone(),
|
||||
&persisted,
|
||||
persisted.disposition.retain_until_unix_nanos - 1,
|
||||
))
|
||||
.await
|
||||
.expect("completed disposition should remain before retention expires")
|
||||
);
|
||||
assert!(
|
||||
Box::pin(garbage_collect_completed_recovery_disposition(
|
||||
store.clone(),
|
||||
&persisted,
|
||||
persisted.disposition.retain_until_unix_nanos,
|
||||
))
|
||||
.await
|
||||
.expect("expired completed disposition should be garbage collected")
|
||||
);
|
||||
assert!(matches!(
|
||||
load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier");
|
||||
assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE");
|
||||
assert!(
|
||||
backend.op_log().await.is_empty(),
|
||||
"legacy disposition must not invoke any backend operation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -19626,330 +19257,6 @@ mod tests {
|
||||
assert_eq!(loaded_ids, intent_ids);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn transition_mutation_measurement(observations: &[TransitionTransactionMutationObservation]) -> (usize, usize, u128) {
|
||||
(
|
||||
observations.len(),
|
||||
observations.iter().map(|observation| observation.encoded_bytes).sum(),
|
||||
observations.iter().map(|observation| observation.elapsed.as_micros()).sum(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn compact_transition_transactions_halve_success_path_quorum_mutations() {
|
||||
let temp_dir = tempfile::tempdir().expect("create transition mutation measurement dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-mutation-measurement", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "MUTATION-MEASURE";
|
||||
register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-mutation-measurement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("measurement bucket should be created");
|
||||
|
||||
let run_case = |profile: &'static str, size: usize| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let object = format!("{profile}-{size}.bin");
|
||||
let mut reader = PutObjReader::from_vec(vec![b'm'; size]);
|
||||
let original = store
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("measurement source should be written");
|
||||
let probe = TransitionTransactionMutationProbe::install(bucket, &object);
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: original.etag.clone().expect("measurement source should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("measurement transition should commit");
|
||||
probe.observations()
|
||||
}
|
||||
};
|
||||
|
||||
let sizes = [4 * 1024, 1024 * 1024];
|
||||
let mut legacy = Vec::with_capacity(sizes.len());
|
||||
for size in sizes {
|
||||
legacy.push((size, run_case("legacy", size).await));
|
||||
}
|
||||
|
||||
let compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
for ((size, legacy_observations), compact_size) in legacy.into_iter().zip(sizes) {
|
||||
assert_eq!(size, compact_size);
|
||||
let compact_observations = run_case("compact", size).await;
|
||||
let legacy_measurement = transition_mutation_measurement(&legacy_observations);
|
||||
let compact_measurement = transition_mutation_measurement(&compact_observations);
|
||||
assert_eq!(legacy_measurement.0, 6, "legacy success should use five saves and one delete");
|
||||
assert_eq!(compact_measurement.0, 3, "compact success should use two saves and one delete");
|
||||
assert!(
|
||||
compact_measurement.1 < legacy_measurement.1,
|
||||
"compact transaction bodies should write fewer aggregate bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
legacy_observations
|
||||
.iter()
|
||||
.map(|observation| (observation.kind, observation.previous_state, observation.state))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(TransitionTransactionMutationKind::Create, None, TransitionTransactionState::UploadStarted),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadStarted),
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadOutcomeUnknown),
|
||||
TransitionTransactionState::Uploaded,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::Uploaded),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::LocalCommitStarted),
|
||||
TransitionTransactionState::Committed,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::Delete,
|
||||
Some(TransitionTransactionState::Committed),
|
||||
TransitionTransactionState::Committed,
|
||||
),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
compact_observations
|
||||
.iter()
|
||||
.map(|observation| (observation.kind, observation.previous_state, observation.state))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(
|
||||
TransitionTransactionMutationKind::Create,
|
||||
None,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::CompareAndSave,
|
||||
Some(TransitionTransactionState::UploadOutcomeUnknown),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
(
|
||||
TransitionTransactionMutationKind::Delete,
|
||||
Some(TransitionTransactionState::LocalCommitStarted),
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
),
|
||||
]
|
||||
);
|
||||
assert!(legacy_observations.iter().all(|observation| observation.succeeded));
|
||||
assert!(compact_observations.iter().all(|observation| observation.succeeded));
|
||||
println!(
|
||||
"transition_mutation_measurement,profile=legacy,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
|
||||
legacy_measurement.0, legacy_measurement.1, legacy_measurement.2, legacy_measurement.0
|
||||
);
|
||||
println!(
|
||||
"transition_mutation_measurement,profile=compact,size={size},mutations={},encoded_bytes={},latency_us={},quorum_operations={}",
|
||||
compact_measurement.0, compact_measurement.1, compact_measurement.2, compact_measurement.0
|
||||
);
|
||||
}
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
let admitted = acquire_transition_transaction_compaction_fleet_proof()
|
||||
.expect("published homogeneous proof should admit one compact writer");
|
||||
assert!(transition_transaction_compaction_fleet_proof_matches(&admitted));
|
||||
drop(compaction_proof);
|
||||
assert!(
|
||||
!transition_transaction_compaction_fleet_proof_matches(&admitted),
|
||||
"revocation must fence a writer admitted by the previous process-epoch snapshot"
|
||||
);
|
||||
drop(admitted);
|
||||
assert!(
|
||||
acquire_transition_transaction_compaction_fleet_proof().is_none(),
|
||||
"revoking the homogeneous proof must restore the legacy writer profile"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn compact_transition_kill_points_preserve_the_only_remote_owner() {
|
||||
let temp_dir = tempfile::tempdir().expect("create compact transition kill-point dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "compact-transition-kill-points", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "COMPACT-KILL";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let _tier_lease = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("mock tier lease should remain available during recovery");
|
||||
let bucket = "compact-transition-kill-points";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("kill-point bucket should be created");
|
||||
let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
|
||||
for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
|
||||
(
|
||||
0,
|
||||
TransitionTransactionKillPoint::PrePutFence,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
(
|
||||
1,
|
||||
TransitionTransactionKillPoint::UploadBeforeCommitFence,
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
1,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
(
|
||||
2,
|
||||
TransitionTransactionKillPoint::LocalCommitBeforeDelete,
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
2,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
(
|
||||
3,
|
||||
TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit,
|
||||
TransitionTransactionState::LocalCommitStarted,
|
||||
2,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let object = format!("kill-point-{index}.bin");
|
||||
let payload = vec![b'k' + u8::try_from(index).expect("small case index should fit u8"); 64 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let source = store
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("kill-point source should be written");
|
||||
let remote_before = backend.object_count().await;
|
||||
let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
|
||||
let transition_store = store.clone();
|
||||
let transition_object = object.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(
|
||||
bucket,
|
||||
&transition_object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("kill-point source should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: source.version_id.map(|version| version.to_string()),
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
transition.abort();
|
||||
assert!(
|
||||
transition
|
||||
.await
|
||||
.expect_err("kill-point transition should be cancelled")
|
||||
.is_cancelled(),
|
||||
"kill-point transition should stop without unwinding"
|
||||
);
|
||||
drop(barrier);
|
||||
|
||||
let transaction = only_transition_transaction(store.clone()).await;
|
||||
assert_eq!((transaction.state, transaction.revision), (expected_state, expected_revision));
|
||||
let paused_source = store
|
||||
.get_object_info(
|
||||
bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("kill-point source metadata should remain readable");
|
||||
assert_eq!(
|
||||
paused_source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE,
|
||||
expected_committed
|
||||
);
|
||||
let expected_remote_at_pause = remote_before + usize::from(point != TransitionTransactionKillPoint::PrePutFence);
|
||||
assert_eq!(backend.object_count().await, expected_remote_at_pause);
|
||||
|
||||
let stats = recover_transition_transaction_records_at(
|
||||
store.clone(),
|
||||
100,
|
||||
None,
|
||||
i128::from(transaction.not_after_unix_nanos) + 1,
|
||||
)
|
||||
.await
|
||||
.expect("kill-point transaction recovery should complete");
|
||||
if expected_recovered {
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 1, 0, 0));
|
||||
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
|
||||
} else {
|
||||
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
|
||||
assert_eq!(
|
||||
transition_transaction_record_count(store.clone()).await,
|
||||
1,
|
||||
"an uncommitted local-commit fence must retain its exact remote owner"
|
||||
);
|
||||
}
|
||||
let expected_remote_after_recovery = if matches!(
|
||||
point,
|
||||
TransitionTransactionKillPoint::LocalCommitBeforeDelete
|
||||
| TransitionTransactionKillPoint::CommitFenceBeforeLocalCommit
|
||||
) {
|
||||
remote_before + 1
|
||||
} else {
|
||||
remote_before
|
||||
};
|
||||
assert_eq!(backend.object_count().await, expected_remote_after_recovery);
|
||||
assert_eq!(backend.remove_count().await, usize::from(index >= 1));
|
||||
|
||||
let mut restored = Vec::new();
|
||||
store
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("kill-point source should remain readable through its authoritative location")
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("kill-point source body should drain");
|
||||
assert_eq!(restored, payload);
|
||||
|
||||
if !expected_recovered {
|
||||
break;
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -20995,7 +20302,7 @@ mod tests {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
data_dir: original.data_dir.expect("source object should have data_dir"),
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: original
|
||||
.mod_time
|
||||
.expect("source object should have mod_time")
|
||||
@@ -21129,7 +20436,7 @@ mod tests {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
data_dir: original.data_dir.expect("source object should have data_dir"),
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: original
|
||||
.mod_time
|
||||
.expect("source object should have mod_time")
|
||||
@@ -21270,77 +20577,10 @@ mod tests {
|
||||
IlmRecoveryClassification::RetainedAmbiguous
|
||||
);
|
||||
let local_commit_control =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
.await
|
||||
.expect("local-commit control should persist");
|
||||
assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
|
||||
let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id)
|
||||
.await
|
||||
.expect("retained upload should be inspectable for a bounded retry");
|
||||
let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id)
|
||||
.await
|
||||
.expect("operator-required local commit should be inspectable for a bounded retry");
|
||||
assert!(upload_status.retry_ready);
|
||||
assert!(local_status.retry_ready);
|
||||
assert!(matches!(
|
||||
retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&upload_started_control_id,
|
||||
upload_status.control_revision + 1,
|
||||
&upload_status.source_generation_sha256,
|
||||
)
|
||||
.await,
|
||||
Err(TransitionOperatorError::StaleRecoveryControl)
|
||||
));
|
||||
|
||||
let put_count_before_retry = backend.put_count().await;
|
||||
let get_count_before_retry = backend.get_count().await;
|
||||
let remove_count_before_retry = backend.remove_count().await;
|
||||
let upload_retry = retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&upload_started_control_id,
|
||||
upload_status.control_revision,
|
||||
&upload_status.source_generation_sha256,
|
||||
)
|
||||
.await
|
||||
.expect("exact retained upload generation should be rearmed");
|
||||
let local_retry = retry_transition_recovery_for_operator(
|
||||
store.clone(),
|
||||
&local_commit_control_id,
|
||||
local_status.control_revision,
|
||||
&local_status.source_generation_sha256,
|
||||
)
|
||||
.await
|
||||
.expect("exact operator-required local commit generation should be rearmed");
|
||||
assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying);
|
||||
assert_eq!(upload_retry.attempt_count, upload_status.attempt_count);
|
||||
assert_eq!(local_retry.attempt_count, local_status.attempt_count);
|
||||
assert_eq!(backend.put_count().await, put_count_before_retry);
|
||||
assert_eq!(backend.get_count().await, get_count_before_retry);
|
||||
assert_eq!(backend.remove_count().await, remove_count_before_retry);
|
||||
assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE");
|
||||
|
||||
let retried = recover_transition_transaction_records(store.clone(), 100, None)
|
||||
.await
|
||||
.expect("rearmed records should be re-evaluated through normal recovery");
|
||||
assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0));
|
||||
let upload_retained =
|
||||
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
|
||||
.await
|
||||
.expect("upload retry result should persist");
|
||||
let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
|
||||
.await
|
||||
.expect("local commit retry result should persist");
|
||||
assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
|
||||
assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired);
|
||||
assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1);
|
||||
assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1);
|
||||
assert_eq!(backend.put_count().await, put_count_before_retry);
|
||||
assert_eq!(backend.get_count().await, get_count_before_retry);
|
||||
assert_eq!(backend.remove_count().await, remove_count_before_retry);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -21601,7 +20841,6 @@ mod tests {
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
@@ -21670,7 +20909,6 @@ mod tests {
|
||||
TransitionTransactionState::UploadOutcomeUnknown,
|
||||
"a response-lost PUT must not remain in UploadStarted"
|
||||
);
|
||||
assert_eq!(transaction.revision, 1, "compact response loss must retain the pre-PUT fence generation");
|
||||
assert!(
|
||||
backend.contains(&transaction.remote_object).await,
|
||||
"the test backend must retain the remote candidate"
|
||||
|
||||
@@ -316,15 +316,10 @@ async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether an empty listing of `prefix` is the proof that lets the caller
|
||||
/// reclaim delete residue under it. Any first page that scanned the whole
|
||||
/// prefix without finding an object or a sub-prefix qualifies, with or without
|
||||
/// a delimiter: that is the shape clients issue when they stat, browse, or
|
||||
/// recursively remove a phantom folder. The purge itself re-verifies every
|
||||
/// directory on every disk before deleting anything.
|
||||
fn should_purge_empty_directory_listing(
|
||||
prefix: &str,
|
||||
marker: Option<&str>,
|
||||
delimiter: Option<&str>,
|
||||
max_keys: i32,
|
||||
incl_deleted: bool,
|
||||
result: &ListObjectsInfo,
|
||||
@@ -332,7 +327,8 @@ fn should_purge_empty_directory_listing(
|
||||
!prefix.is_empty()
|
||||
&& prefix.ends_with(SLASH_SEPARATOR)
|
||||
&& marker.is_none()
|
||||
&& max_keys > 0
|
||||
&& delimiter.is_none_or(str::is_empty)
|
||||
&& max_keys == 1
|
||||
&& !incl_deleted
|
||||
&& !result.is_truncated
|
||||
&& result.objects.is_empty()
|
||||
@@ -3851,10 +3847,16 @@ impl ECStore {
|
||||
.list_objects_from_opt_in_key_only_provider(&opts, mode, max_keys, incl_deleted)
|
||||
.await?
|
||||
{
|
||||
if should_purge_empty_directory_listing(prefix, opts.marker.as_deref(), max_keys, incl_deleted, &result)
|
||||
&& has_authoritative_never_versioned_state_in(&self.ctx, bucket)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
if should_purge_empty_directory_listing(
|
||||
prefix,
|
||||
opts.marker.as_deref(),
|
||||
delimiter.as_deref(),
|
||||
max_keys,
|
||||
incl_deleted,
|
||||
&result,
|
||||
) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
self.purge_orphan_dir_object(bucket, prefix).await;
|
||||
}
|
||||
@@ -3941,10 +3943,16 @@ impl ECStore {
|
||||
objects,
|
||||
prefixes,
|
||||
};
|
||||
if should_purge_empty_directory_listing(prefix, opts.marker.as_deref(), max_keys, incl_deleted, &result)
|
||||
&& has_authoritative_never_versioned_state_in(&self.ctx, bucket)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
if should_purge_empty_directory_listing(
|
||||
prefix,
|
||||
opts.marker.as_deref(),
|
||||
delimiter.as_deref(),
|
||||
max_keys,
|
||||
incl_deleted,
|
||||
&result,
|
||||
) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
self.purge_orphan_dir_object(bucket, prefix).await;
|
||||
}
|
||||
@@ -8835,28 +8843,26 @@ mod test {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_directory_listing_purge_requires_complete_first_page_of_prefix() {
|
||||
fn empty_directory_listing_purge_requires_complete_exact_recursive_request() {
|
||||
let empty = ListObjectsInfo::default();
|
||||
assert!(should_purge_empty_directory_listing("ghost/", None, 1, false, &empty));
|
||||
assert!(should_purge_empty_directory_listing("ghost/", None, 1000, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost", None, 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", Some("marker"), 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, 0, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, 1, true, &empty));
|
||||
assert!(should_purge_empty_directory_listing("ghost/", None, None, 1, false, &empty));
|
||||
assert!(should_purge_empty_directory_listing("ghost/", None, Some(""), 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost", None, None, 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", Some("marker"), None, 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, Some("/"), 1, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, None, 0, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, None, 2, false, &empty));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, true, &empty));
|
||||
|
||||
let mut live = ListObjectsInfo::default();
|
||||
live.objects.push(ObjectInfo::default());
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, 1, false, &live));
|
||||
|
||||
let mut prefixed = ListObjectsInfo::default();
|
||||
prefixed.prefixes.push("ghost/child/".to_owned());
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, 1, false, &prefixed));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &live));
|
||||
|
||||
let truncated = ListObjectsInfo {
|
||||
is_truncated: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, 1, false, &truncated));
|
||||
assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &truncated));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -8910,59 +8916,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_hides_and_purges_committed_delete_residue() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-delimiter-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let residue = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics")
|
||||
.join("2026")
|
||||
.join("object.parquet")
|
||||
.join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
}
|
||||
|
||||
// The object directory holds only a deleted version's data dir, so a
|
||||
// console-style browse of its parent must not show it as a folder.
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/2026/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "delete residue must not surface as a prefix");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path().join(bucket).join("metrics").join("2026").exists(),
|
||||
"the empty delimiter listing should reclaim the committed delete residue under it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::walker_key_only();
|
||||
|
||||
@@ -442,7 +442,7 @@ pub(crate) mod utils;
|
||||
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
@@ -1787,7 +1787,7 @@ mod tests {
|
||||
|
||||
// Build a minimal ECStore carrying an explicit instance context. Empty
|
||||
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
|
||||
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||
let endpoint_pools = EndpointServerPools::default();
|
||||
Arc::new(ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::bucket::utils::has_bad_path_component;
|
||||
use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use tracing::{debug, error};
|
||||
|
||||
@@ -22,6 +25,203 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
|
||||
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
|
||||
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
|
||||
|
||||
/// An instance-bound capability for internal writes before ECStore/IAM startup.
|
||||
/// Its private context and volume checks cannot be replaced by a caller guard.
|
||||
#[derive(Clone)]
|
||||
pub struct BootstrapLocalTarget {
|
||||
ctx: Arc<InstanceContext>,
|
||||
}
|
||||
|
||||
impl BootstrapLocalTarget {
|
||||
pub fn new(ctx: Arc<InstanceContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub fn is_for_store(&self, store: &ECStore) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, &store.ctx)
|
||||
}
|
||||
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if scanner_token.is_some() {
|
||||
return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
|
||||
}
|
||||
validate_bootstrap_volume(source.0)?;
|
||||
validate_bootstrap_volume(destination.0)?;
|
||||
rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
validate_bootstrap_volume(volume)?;
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
// Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
|
||||
// Validate both raw rename volumes before any disk lookup or admission.
|
||||
if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Execute on this instance's active local disk through the physical owner.
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
scanner_token: Option<Uuid>,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
|
||||
Some(Arc::new(
|
||||
self.acquire_scanner_publication_lease_guard(token)
|
||||
.await
|
||||
.map_err(|err| DiskError::other(err.to_string()))?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn undo_local_write(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
// The optional ID is a cold lookup to cache only after final admission.
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
|
||||
if let Some(path) = cached_path {
|
||||
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
&& disk.get_disk_id().await? == Some(disk_id)
|
||||
{
|
||||
return Ok((disk, None));
|
||||
}
|
||||
}
|
||||
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
|
||||
// Disk identity may perform format I/O. No registry guard spans this await.
|
||||
for disk in disks {
|
||||
if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
|
||||
return Ok((disk, Some(disk_id)));
|
||||
}
|
||||
}
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
volume: &str,
|
||||
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
|
||||
if !matches!(disk.as_ref(), Disk::Local(_)) {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
let map = ctx.local_disk_map();
|
||||
let active = map.read().await;
|
||||
if !active
|
||||
.get(&disk.endpoint().to_string())
|
||||
.and_then(Option::as_ref)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, disk))
|
||||
{
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
// Preserve registry -> ID-cache lock order; no filesystem I/O under either.
|
||||
if let Some(disk_id) = disk_id {
|
||||
ctx.local_disk_id_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk_id, disk.endpoint().to_string());
|
||||
}
|
||||
// Admission linearizes under the registry read: replacement/quarantine
|
||||
// before this point rejects; later changes do not revoke physical I/O.
|
||||
Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit()))
|
||||
}
|
||||
|
||||
async fn rename_local_data_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?;
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn undo_local_write_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
) -> DiskResult<()> {
|
||||
if !opts.undo_write {
|
||||
return Err(DiskError::other("target undo requires undo_write"));
|
||||
}
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?;
|
||||
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
|
||||
.await;
|
||||
drop(owner);
|
||||
result
|
||||
}
|
||||
|
||||
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
|
||||
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
|
||||
}
|
||||
@@ -265,6 +465,522 @@ mod tests {
|
||||
}])
|
||||
}
|
||||
|
||||
async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
|
||||
let mut format = crate::layout::format::FormatV3::new(1, 1);
|
||||
format.erasure.this = id;
|
||||
format.erasure.sets[0][0] = id;
|
||||
let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("create format volume");
|
||||
tokio::fs::write(
|
||||
meta.join(crate::disk::FORMAT_CONFIG_FILE),
|
||||
serde_json::to_vec(&format).expect("encode format"),
|
||||
)
|
||||
.await
|
||||
.expect("write real disk identity");
|
||||
let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open real local disk");
|
||||
assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
disk
|
||||
}
|
||||
|
||||
fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 1, 0);
|
||||
fi.erasure.index = 1;
|
||||
fi.version_id = Some(version);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.size = i64::try_from(body.len()).expect("fixture length");
|
||||
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||
number: 1,
|
||||
size: body.len(),
|
||||
actual_size: fi.size,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.data = Some(bytes::Bytes::from_static(body));
|
||||
fi.set_inline_data();
|
||||
fi
|
||||
}
|
||||
|
||||
async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
|
||||
let dir = disk.path().join(volume);
|
||||
tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
|
||||
disk.write_metadata(volume, volume, object, fi.clone())
|
||||
.await
|
||||
.expect("seed real metadata");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
object,
|
||||
&fi.version_id.expect("fixture version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read fixture before mutation");
|
||||
assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
|
||||
tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
let ctx_a = Arc::new(InstanceContext::new());
|
||||
let ctx_b = Arc::new(InstanceContext::new());
|
||||
let a = tempfile::tempdir().expect("A root");
|
||||
let b = tempfile::tempdir().expect("B root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk_a = target_disk(&ctx_a, a.path(), id).await;
|
||||
let disk_b = target_disk(&ctx_b, b.path(), id).await;
|
||||
if warm {
|
||||
assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
|
||||
assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
|
||||
}
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("destination", version, b"new-A");
|
||||
for disk in [&disk_a, &disk_b] {
|
||||
seed_target(disk, "target-bucket", "staged", fi.clone()).await;
|
||||
}
|
||||
let b_before = seed_target(
|
||||
&disk_b,
|
||||
"target-bucket",
|
||||
"destination",
|
||||
target_file_info("destination", version, b"old-B"),
|
||||
)
|
||||
.await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
|
||||
store
|
||||
.rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
|
||||
.await
|
||||
.expect("rename on A");
|
||||
let read = disk_a
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read committed A");
|
||||
assert_eq!(read.data, fi.data, "warm={warm}");
|
||||
assert_eq!(
|
||||
tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
|
||||
.await
|
||||
.expect("B metadata"),
|
||||
b_before
|
||||
);
|
||||
assert!(b.path().join("target-bucket/staged/xl.meta").exists());
|
||||
assert!(ctx_a.namespace_commit_generation() > 0);
|
||||
assert_eq!(ctx_b.namespace_commit_generation(), 0);
|
||||
assert!(!ctx_a.namespace_commits_pending());
|
||||
assert!(!ctx_b.namespace_commits_pending());
|
||||
assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
for state in ["removed", "quarantined", "replaced"] {
|
||||
let replacement = new_disk(
|
||||
&disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("separate active Arc");
|
||||
let map = ctx.local_disk_map();
|
||||
let mut entries = map.write().await;
|
||||
match state {
|
||||
"removed" => {
|
||||
entries.remove(&endpoint);
|
||||
}
|
||||
"quarantined" => {
|
||||
entries.insert(endpoint.clone(), None);
|
||||
}
|
||||
_ => {
|
||||
entries.insert(endpoint.clone(), Some(replacement));
|
||||
}
|
||||
}
|
||||
drop(entries);
|
||||
assert!(
|
||||
matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)),
|
||||
"{state}"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let old_id = Uuid::new_v4();
|
||||
let old = target_disk(&ctx, root.path(), old_id).await;
|
||||
assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
|
||||
let replacement_id = Uuid::new_v4();
|
||||
let replacement = target_disk(&ctx, root.path(), replacement_id).await;
|
||||
assert!(!Arc::ptr_eq(&old, &replacement));
|
||||
assert!(matches!(
|
||||
local_disk_candidate(&ctx, &old_id.to_string()).await,
|
||||
Err(DiskError::DiskNotFound)
|
||||
));
|
||||
let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
|
||||
.await
|
||||
.expect("replacement UUID");
|
||||
assert!(Arc::ptr_eq(&candidate, &replacement));
|
||||
assert_eq!(verified, Some(replacement_id));
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let target = BootstrapLocalTarget::new(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"body");
|
||||
let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
|
||||
let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
|
||||
for invalid in [
|
||||
"victim",
|
||||
".rustfs.sys/../victim",
|
||||
".rustfs.sys/./tmp",
|
||||
".rustfs.sys/ .. /victim",
|
||||
".rustfs.sys\\..\\victim",
|
||||
".minio.sys/../victim",
|
||||
] {
|
||||
for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
|
||||
.await
|
||||
.is_err(),
|
||||
"src={src}, dst={dst}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.undo_local_write(
|
||||
&disk.endpoint().to_string(),
|
||||
invalid,
|
||||
"staged",
|
||||
fi.clone(),
|
||||
DeleteOptions {
|
||||
undo_write: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"{invalid}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
target
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
(".rustfs.sys/tmp", "staged"),
|
||||
&fi,
|
||||
(".rustfs.sys/tmp", "destination"),
|
||||
Some(Uuid::new_v4())
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("victim/staged/xl.meta"))
|
||||
.await
|
||||
.expect("user source"),
|
||||
user_before
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
|
||||
.await
|
||||
.expect("metadata source"),
|
||||
meta_before
|
||||
);
|
||||
assert!(!root.path().join("victim/destination").exists());
|
||||
assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
|
||||
for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
|
||||
seed_target(&disk, volume, "staged", fi.clone()).await;
|
||||
BootstrapLocalTarget::new(ctx.clone())
|
||||
.rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
|
||||
.await
|
||||
.expect("legitimate bootstrap metadata write");
|
||||
let read = disk
|
||||
.read_version(
|
||||
volume,
|
||||
volume,
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read bootstrap result");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let sibling = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let (token, _) = store
|
||||
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||
.await
|
||||
.expect("real scanner token in A");
|
||||
let destination = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local disk")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&destination, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token),
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical rename entered"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
drop(rename);
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(!sibling.namespace_commits_pending());
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
|
||||
.await
|
||||
.is_err(),
|
||||
"real pending rename blocks another scanner publication"
|
||||
);
|
||||
assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
|
||||
let gate = ctx.data_movement_operation_gate();
|
||||
assert!(
|
||||
gate.clone().try_write_owned().is_err(),
|
||||
"physical operation still owns the scanner read guard"
|
||||
);
|
||||
drop(release_tx);
|
||||
let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
|
||||
.await
|
||||
.expect("physical tail must release scanner guard");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("namespace owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual late commit");
|
||||
assert_eq!(read.data, fi.data);
|
||||
assert!(ctx.namespace_commit_generation() >= 2);
|
||||
assert_eq!(sibling.namespace_commit_generation(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let other = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let other_store = super::super::tests::build_store_with_ctx(other);
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
|
||||
let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
|
||||
let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
|
||||
let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
|
||||
assert!(store.release_scanner_publication_lease(released).await);
|
||||
let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
|
||||
for token in [Uuid::new_v4(), foreign, released] {
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(token)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
store
|
||||
.rename_local_data(
|
||||
&disk.endpoint().to_string(),
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
Some(valid)
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"expired real token"
|
||||
);
|
||||
let _ = other_store.release_scanner_publication_lease(foreign).await;
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
|
||||
.await
|
||||
.expect("source bytes"),
|
||||
before
|
||||
);
|
||||
assert!(!root.path().join("target-bucket/destination").exists());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
|
||||
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||
let path = disk
|
||||
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||
.expect("local")
|
||||
.expect("destination IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = hooks::install(&path, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let mut rename = Box::pin(store.rename_local_data(
|
||||
&disk_ref,
|
||||
("target-bucket", "staged"),
|
||||
&fi,
|
||||
("target-bucket", "destination"),
|
||||
None,
|
||||
));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
result = &mut rename => panic!("completed before physical pause: {result:?}"),
|
||||
entered = entered_rx => entered.expect("physical entry"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded entry");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||
.await
|
||||
.expect("ordinary deadline remains enabled");
|
||||
assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
|
||||
drop(rename);
|
||||
assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late physical owner drains");
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"destination",
|
||||
&fi.version_id.expect("version").to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read actual timeout tail");
|
||||
assert_eq!(read.data, fi.data);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||
|
||||
@@ -692,15 +692,10 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
// The version stays on disk while the purge replicates
|
||||
// (status PENDING/FAILED); its data dir must stay with
|
||||
// it. Returning the dir here made the disk layer delete
|
||||
// it, which turned every non-inline retained version
|
||||
// into an unreadable zombie: the purge state could never
|
||||
// be applied and the bucket could never be deleted.
|
||||
let old_dir = v.object.as_ref().map(|v| v.data_dir).unwrap_or_default();
|
||||
self.set_idx(i, v)?;
|
||||
|
||||
return Ok(None);
|
||||
return Ok(old_dir);
|
||||
}
|
||||
found_index = Some(i);
|
||||
}
|
||||
@@ -2707,58 +2702,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for rustfs/backlog#2340: a version purge that still awaits
|
||||
/// the replication target keeps the object version on disk with a pending
|
||||
/// purge status. Its data dir must be retained with it; handing the dir
|
||||
/// back here made the disk layer delete it, leaving every non-inline
|
||||
/// retained version unreadable. The dir is released only once the purge
|
||||
/// completes and the version itself goes away.
|
||||
#[test]
|
||||
fn delete_version_pending_version_purge_retains_object_data_dir() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut fm = FileMeta::new();
|
||||
let mut fi = FileInfo::new("object", 2, 2);
|
||||
fi.version_id = Some(version_id);
|
||||
fi.data_dir = Some(data_dir);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fm.add_version(fi).unwrap();
|
||||
|
||||
let pending_purge = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
mark_deleted: true,
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=PENDING;".to_string()),
|
||||
purge_targets: version_purge_statuses_map("target=PENDING;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let freed = fm.delete_version(&pending_purge).unwrap();
|
||||
assert_eq!(freed, None, "a pending purge must not release the retained version's data dir");
|
||||
assert_eq!(fm.versions.len(), 1, "the version must stay until the purge replicates");
|
||||
let retained = fm
|
||||
.into_fileinfo("vol", "object", &version_id.to_string(), false, false, true)
|
||||
.unwrap();
|
||||
assert_eq!(retained.data_dir, Some(data_dir));
|
||||
assert_eq!(retained.version_purge_status(), VersionPurgeStatusType::Pending);
|
||||
|
||||
let completed_purge = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
replication_state_internal: Some(ReplicationState {
|
||||
version_purge_status_internal: Some("target=COMPLETE;".to_string()),
|
||||
purge_targets: version_purge_statuses_map("target=COMPLETE;"),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let freed = fm.delete_version(&completed_purge).unwrap();
|
||||
assert_eq!(freed, Some(data_dir), "a completed purge removes the version and releases its data dir");
|
||||
assert!(fm.versions.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_accepts_delete_only_marker_and_free_version_paths() {
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
|
||||
@@ -442,10 +442,6 @@ pub struct ReplicatedTargetInfo {
|
||||
pub error: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target_delete_marker_version_id: Option<String>,
|
||||
/// Kept in step with the replication crate's copy: the id a target that
|
||||
/// mints its own version ids assigned to this object version.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target_version_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicatedTargetInfo {
|
||||
|
||||
+19
-155
@@ -76,8 +76,6 @@ struct HealTaskStatusPayload<'a> {
|
||||
min_seq: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
progress: Option<&'a HealProgress>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
outcome: Option<&'a super::outcome::HealTaskOutcome>,
|
||||
}
|
||||
|
||||
fn u64_is_zero(value: &u64) -> bool {
|
||||
@@ -89,18 +87,17 @@ fn encode_heal_task_status_payload(
|
||||
mut items: Vec<HealResultItem>,
|
||||
progress: Option<&HealProgress>,
|
||||
mut truncated: bool,
|
||||
sequence: (u64, u64),
|
||||
outcome: Option<&super::outcome::HealTaskOutcome>,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
) -> Result<(Vec<u8>, bool)> {
|
||||
loop {
|
||||
let data = serde_json::to_vec(&HealTaskStatusPayload {
|
||||
summary,
|
||||
items: &items,
|
||||
truncated,
|
||||
next_seq: sequence.0,
|
||||
min_seq: sequence.1,
|
||||
next_seq,
|
||||
min_seq,
|
||||
progress,
|
||||
outcome,
|
||||
})
|
||||
.map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
|
||||
if data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE {
|
||||
@@ -114,21 +111,25 @@ fn encode_heal_task_status_payload(
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
|
||||
if !truncated {
|
||||
return detail;
|
||||
}
|
||||
let truncation = "heal result items were truncated";
|
||||
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
|
||||
}
|
||||
|
||||
fn encode_heal_status_response(
|
||||
summary: &str,
|
||||
items: Vec<HealResultItem>,
|
||||
progress: Option<&HealProgress>,
|
||||
detail: Option<String>,
|
||||
truncated: bool,
|
||||
sequence: (u64, u64),
|
||||
outcome: Option<&super::outcome::HealTaskOutcome>,
|
||||
next_seq: u64,
|
||||
min_seq: u64,
|
||||
) -> Result<(Vec<u8>, Option<String>)> {
|
||||
let (summary, detail) = match outcome {
|
||||
Some(outcome) => outcome.legacy_status(summary, detail),
|
||||
None => (summary, detail),
|
||||
};
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, sequence, outcome)?;
|
||||
Ok((data, super::outcome::heal_status_detail(detail, truncated)))
|
||||
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?;
|
||||
Ok((data, heal_status_detail(detail, truncated)))
|
||||
}
|
||||
|
||||
impl HealChannelProcessor {
|
||||
@@ -438,7 +439,6 @@ impl HealChannelProcessor {
|
||||
.await
|
||||
};
|
||||
|
||||
let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
|
||||
let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report {
|
||||
Ok(HealTaskReport {
|
||||
status: HealTaskStatus::Pending | HealTaskStatus::Running,
|
||||
@@ -576,15 +576,8 @@ impl HealChannelProcessor {
|
||||
}
|
||||
};
|
||||
|
||||
let (data, detail) = encode_heal_status_response(
|
||||
&summary,
|
||||
items,
|
||||
progress.as_ref(),
|
||||
detail,
|
||||
truncated,
|
||||
(next_seq, min_seq),
|
||||
outcome.as_deref(),
|
||||
)?;
|
||||
let (data, detail) =
|
||||
encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?;
|
||||
|
||||
let response = HealChannelResponse {
|
||||
request_id: client_token,
|
||||
@@ -873,7 +866,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false, (0, 0), None).unwrap();
|
||||
let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap();
|
||||
|
||||
assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
|
||||
let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
|
||||
@@ -882,135 +875,6 @@ mod tests {
|
||||
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_fixture_matches_canonical_owner_and_preserves_legacy_terminals() {
|
||||
use crate::heal::outcome::*;
|
||||
let cases: serde_json::Value = serde_json::from_str(include_str!("../../../madmin/tests/fixtures/heal-outcome-v3.json"))
|
||||
.expect("shared client fixtures");
|
||||
for case in cases.as_array().expect("fixture cases") {
|
||||
if case.get("remoteResponse").is_some() {
|
||||
continue;
|
||||
}
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
let name = case["name"].as_str().expect("case name");
|
||||
if matches!(name, "unknown" | "completed_with_errors") {
|
||||
let disposition = if name == "unknown" {
|
||||
HealObjectDisposition::Unknown
|
||||
} else {
|
||||
outcome.attempt_failed();
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
}
|
||||
let abort = match name {
|
||||
"cancelled" => Some(HealAbortReason::Cancelled),
|
||||
"deadline" => Some(HealAbortReason::Deadline),
|
||||
"untraversable" => Some(HealAbortReason::Untraversable),
|
||||
_ => None,
|
||||
};
|
||||
outcome.finish(abort);
|
||||
let expected = &case["response"];
|
||||
let initial_detail = abort.map(|reason| {
|
||||
match reason {
|
||||
HealAbortReason::Cancelled => "heal task cancelled",
|
||||
HealAbortReason::Deadline => "heal task timed out",
|
||||
HealAbortReason::Untraversable => "heal listing is untraversable",
|
||||
}
|
||||
.to_string()
|
||||
});
|
||||
let (bytes, detail) = encode_heal_status_response(
|
||||
if abort.is_some() { "stopped" } else { "finished" },
|
||||
Vec::new(),
|
||||
None,
|
||||
initial_detail,
|
||||
true,
|
||||
(9, 4),
|
||||
Some(&outcome),
|
||||
)
|
||||
.expect("canonical owner encoding");
|
||||
let decoded: serde_json::Value = serde_json::from_slice(&bytes).expect("wire payload");
|
||||
assert_eq!(decoded["summary"], expected["summary"], "{name}");
|
||||
assert_eq!(detail.unwrap_or_default(), expected["detail"].as_str().expect("detail"), "{name}");
|
||||
assert_eq!(decoded["outcome"], expected["outcome"], "{name}");
|
||||
assert_eq!((decoded["next_seq"].as_u64(), decoded["min_seq"].as_u64()), (Some(9), Some(4)));
|
||||
assert!(decoded["outcome"].get("retainedObjectBytes").is_none());
|
||||
assert!(decoded["outcome"].get("untraversable").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_abort_cannot_be_hidden_by_a_finished_status() {
|
||||
use crate::heal::outcome::{HealAbortReason, HealTaskOutcome};
|
||||
for reason in [
|
||||
HealAbortReason::Cancelled,
|
||||
HealAbortReason::Deadline,
|
||||
HealAbortReason::Untraversable,
|
||||
] {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.finish(Some(reason));
|
||||
let (data, detail) = encode_heal_status_response("finished", Vec::new(), None, None, false, (0, 0), Some(&outcome))
|
||||
.expect("canonical abort adapter");
|
||||
let json: serde_json::Value = serde_json::from_slice(&data).expect("public state");
|
||||
assert_eq!(json["summary"], "stopped");
|
||||
assert_eq!(json["outcome"]["execution"]["state"], "aborted");
|
||||
assert!(detail.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_payload_bound_keeps_cumulative_outcome_and_cursors() {
|
||||
use crate::heal::outcome::*;
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
for index in 0..256 {
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: format!("object-{index}"),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
detail: Some("\"".repeat(1024)),
|
||||
});
|
||||
}
|
||||
let retained = outcome.objects.len();
|
||||
let items = vec![
|
||||
HealResultItem::default(),
|
||||
HealResultItem {
|
||||
detail: "x".repeat(MAX_HEAL_STATUS_PAYLOAD_SIZE + 1),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let (bytes, detail) = encode_heal_status_response("running", items, None, None, false, (9, 4), Some(&outcome))
|
||||
.expect("bounded status with cumulative outcome");
|
||||
assert!(bytes.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
|
||||
let wire: serde_json::Value = serde_json::from_slice(&bytes).expect("bounded payload");
|
||||
assert_eq!(wire["items"].as_array().expect("items").len(), 1);
|
||||
assert_eq!(wire["truncated"], true);
|
||||
assert_eq!((wire["next_seq"].as_u64(), wire["min_seq"].as_u64()), (Some(9), Some(4)));
|
||||
assert_eq!(wire["outcome"]["counters"]["processed"], 256);
|
||||
assert_eq!(wire["outcome"]["counters"]["healed"], 0);
|
||||
assert_eq!(wire["outcome"]["objects"].as_array().expect("outcome window").len(), retained);
|
||||
assert!(retained < 256 && outcome.objects_truncated);
|
||||
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_response_preserves_all_admission_outcomes() {
|
||||
let cases = [
|
||||
|
||||
@@ -150,7 +150,6 @@ impl HealManager {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
&& !super::super::replacement_readiness::directory_backed_replacement_fallback_enabled()
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
|
||||
@@ -313,6 +313,7 @@ impl HealManager {
|
||||
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
@@ -374,11 +375,11 @@ impl HealManager {
|
||||
drop(stats);
|
||||
if terminal_completion {
|
||||
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
|
||||
// Neither task status nor the diagnostic outcome
|
||||
// window supplies a storage-owned repair receipt.
|
||||
// Release only the ingress lease for rediscovery;
|
||||
// preserve the producer's existing retry hints.
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
if successful_completion {
|
||||
emit_mrf_repaired_events(notice_targets);
|
||||
} else {
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -705,6 +706,20 @@ fn move_mrf_repair_notice_targets(
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
@@ -731,8 +746,11 @@ pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
|
||||
completed_heals
|
||||
.retain(|_, completed| now.duration_since(completed.completed_at).unwrap_or_default() <= KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
completed_heals.retain(|_, completed| {
|
||||
now.duration_since(completed.completed_at)
|
||||
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
|
||||
key.capacity()
|
||||
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
|
||||
|
||||
@@ -189,104 +189,37 @@ fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
|
||||
);
|
||||
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(entries.contains_key("ttl-boundary"));
|
||||
assert!(entries.contains_key("future"), "clock rollback must not expire a new completion");
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert!(entries.contains_key("future"));
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1) + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert!(entries.contains_key("future"), "the exact TTL boundary remains retained");
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(2) + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert!(entries.contains_key("ttl-boundary"));
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_clock_rollback_preserves_terminal_alias_queries() {
|
||||
let completed_at = SystemTime::now() + Duration::from_secs(3600);
|
||||
for status in [
|
||||
HealTaskStatus::Completed,
|
||||
HealTaskStatus::Failed {
|
||||
error: "fixture failure".to_string(),
|
||||
},
|
||||
HealTaskStatus::Cancelled,
|
||||
] {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut snapshot = completed_retention_fixture(completed_at);
|
||||
snapshot.status = status.clone();
|
||||
let expected_progress = snapshot.progress.clone();
|
||||
let snapshot = Arc::new(snapshot);
|
||||
{
|
||||
let mut completed = manager.completed_heals.lock().await;
|
||||
completed.insert("canonical".to_string(), Arc::clone(&snapshot));
|
||||
completed.insert("alias".to_string(), Arc::clone(&snapshot));
|
||||
}
|
||||
for token in ["canonical", "alias"] {
|
||||
let report = manager
|
||||
.get_task_report_since(token, Some(3))
|
||||
.await
|
||||
.expect("a clock rollback must retain terminal queries");
|
||||
assert_eq!(report.status, status);
|
||||
assert_eq!(report.progress, expected_progress);
|
||||
assert_eq!(report.result_items.len(), 1);
|
||||
assert_eq!((report.min_seq, report.next_seq), (3, 5));
|
||||
assert!(!report.result_items_truncated);
|
||||
}
|
||||
let mut completed = manager.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses_at(&mut completed, completed_at + KEEP_HEAL_TASK_STATUS_DURATION);
|
||||
assert_eq!(completed.len(), 2, "both tokens remain at the exact TTL boundary");
|
||||
prune_completed_heal_statuses_at(&mut completed, completed_at + KEEP_HEAL_TASK_STATUS_DURATION + Duration::from_nanos(1));
|
||||
assert!(completed.is_empty(), "both tokens expire after the TTL");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_clock_rollback_keeps_count_and_alias_eviction_bounded() {
|
||||
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(3600);
|
||||
let oldest = Arc::new(completed_retention_fixture(now + Duration::from_secs(1)));
|
||||
let mut entries = HashMap::from([
|
||||
("oldest".to_string(), Arc::clone(&oldest)),
|
||||
("oldest-alias".to_string(), oldest),
|
||||
]);
|
||||
for index in 2..=MAX_COMPLETED_HEAL_TOKENS {
|
||||
entries.insert(
|
||||
format!("task-{index}"),
|
||||
Arc::new(completed_retention_fixture(now + Duration::from_secs(2))),
|
||||
);
|
||||
}
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
|
||||
assert!(!entries.contains_key("oldest"));
|
||||
assert!(!entries.contains_key("oldest-alias"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_total_byte_cap_and_cap_plus_one() {
|
||||
let now = SystemTime::now();
|
||||
for completed_at in [now, now + Duration::from_secs(1)] {
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(completed_at);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(now);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -944,84 +877,6 @@ fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSourc
|
||||
request
|
||||
}
|
||||
|
||||
fn scoped_object_request(bucket: &str, object: &str, pool_index: usize, set_index: usize) -> HealRequest {
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_bulkhead_starts_other_sets_and_retains_same_set_tail() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
{
|
||||
let mut config = manager.config.write().await;
|
||||
config.max_concurrent_heals = 2;
|
||||
config.max_concurrent_per_set = 1;
|
||||
config.set_bulkhead_enable = true;
|
||||
config.event_driven_scheduler_enable = false;
|
||||
config.mainline_throttle_enable = false;
|
||||
}
|
||||
|
||||
let first_set = scoped_object_request("scheduler-bulkhead-set-a-first", "object-a", 0, 1);
|
||||
let first_set_id = first_set.id.clone();
|
||||
let same_set_tail = scoped_object_request("scheduler-bulkhead-set-a-tail", "object-b", 0, 1);
|
||||
let same_set_tail_id = same_set_tail.id.clone();
|
||||
let other_set = scoped_object_request("scheduler-bulkhead-set-b", "object-c", 0, 2);
|
||||
let other_set_id = other_set.id.clone();
|
||||
|
||||
let first_hook = Arc::new(CompletedRetentionHook::default());
|
||||
let other_hook = Arc::new(CompletedRetentionHook::default());
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert("scheduler-bulkhead-set-a-first".to_string(), first_hook.clone());
|
||||
hooks.insert("scheduler-bulkhead-set-b".to_string(), other_hook.clone());
|
||||
}
|
||||
{
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.push(first_set), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(same_set_tail), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(other_set), QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), first_hook.started.notified())
|
||||
.await
|
||||
.expect("first set task should start");
|
||||
tokio::time::timeout(Duration::from_secs(5), other_hook.started.notified())
|
||||
.await
|
||||
.expect("other set task should start despite same-set tail");
|
||||
|
||||
assert_eq!(manager.get_active_task_count().await, 2);
|
||||
assert_eq!(manager.get_queue_length().await, 1);
|
||||
assert!(matches!(manager.get_task_status(&same_set_tail_id).await, Ok(HealTaskStatus::Pending)));
|
||||
{
|
||||
let active = manager.active_heals.lock().await;
|
||||
assert!(active.contains_key(&first_set_id));
|
||||
assert!(active.contains_key(&other_set_id));
|
||||
assert!(!active.contains_key(&same_set_tail_id));
|
||||
let counts = running_heal_set_counts(&active);
|
||||
assert_eq!(counts.get("pool_0_set_1"), Some(&1));
|
||||
assert_eq!(counts.get("pool_0_set_2"), Some(&1));
|
||||
}
|
||||
|
||||
manager.cancel_task(&first_set_id).await.expect("cancel first active task");
|
||||
manager.cancel_task(&other_set_id).await.expect("cancel other active task");
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|bucket, _| !bucket.starts_with("scheduler-bulkhead-"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_push_displacing_lower_priority_actually_enqueues_new_request() {
|
||||
// Regression for the release-build defect where the enqueue side effect lived inside
|
||||
@@ -3241,7 +3096,7 @@ async fn test_cancel_task_removes_queued_request() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
|
||||
async fn test_mrf_repaired_notice_waits_for_successful_completion() {
|
||||
let bucket = "mrf-completion-success";
|
||||
let object = "object";
|
||||
let version_id = Some([9u8; 16]);
|
||||
@@ -3271,81 +3126,21 @@ async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
|
||||
);
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let stats = manager.get_statistics().await;
|
||||
if stats.successful_tasks + stats.failed_tasks > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
for _ in 0..100 {
|
||||
let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
if !events.is_empty() {
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].object.as_ref(), object);
|
||||
assert_eq!(events[0].version_id, version_id);
|
||||
return;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("scheduler completes the task");
|
||||
assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
|
||||
assert!(!lock_mrf_repair_notice_targets(&manager.mrf_repair_notice_targets).contains_key(&receipt.task_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() {
|
||||
for empty_window in [false, true] {
|
||||
let bucket = if empty_window {
|
||||
"mrf-empty-outcome"
|
||||
} else {
|
||||
"mrf-dry-run-outcome"
|
||||
};
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::new(
|
||||
if empty_window {
|
||||
HealType::Cluster
|
||||
} else {
|
||||
HealType::Object {
|
||||
bucket: bucket.to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
}
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run: !empty_window,
|
||||
recreate_missing: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt(request, Arc::from(bucket), Arc::from("object"), None)
|
||||
.await
|
||||
.expect("notice target registered");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let stats = manager.get_statistics().await;
|
||||
if stats.successful_tasks + stats.failed_tasks > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("scheduler completed");
|
||||
let report = manager.get_task_report(&receipt.task_id).await.expect("completed report");
|
||||
assert_eq!(report.status, HealTaskStatus::Completed);
|
||||
let outcome = report.outcome.expect("canonical outcome");
|
||||
if empty_window {
|
||||
assert!(outcome.objects.is_empty());
|
||||
} else {
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
crate::heal::outcome::HealObjectDisposition::DryRunObserved
|
||||
);
|
||||
}
|
||||
assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("successful MRF-owned heal should emit one repaired event");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
|
||||
async fn test_mrf_repaired_notice_removed_on_queued_cancel_without_event() {
|
||||
let bucket = "mrf-completion-cancel";
|
||||
let object = "object";
|
||||
let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
|
||||
@@ -25,13 +25,12 @@
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Neither ingress nor manager admission is a durable ownership
|
||||
//! receipt. The last flush window can be lost. Read-repair can rediscover a
|
||||
//! failed read; the scanner retains bounded, expiring retry hints. Partial
|
||||
//! writes also use a best-effort in-memory fast path, not a durable successor.
|
||||
//! These mechanisms must not be reported as verified repair completion.
|
||||
//! The partial-write caller's restart-survival requirement remains unmet by
|
||||
//! admission alone; a verified durable handoff is still required.
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable because
|
||||
//! every producer keeps its own safety net: read-repair re-detects on the
|
||||
//! next failing read, and the scanner's corrupt-metadata branch leaves a
|
||||
//! pending-ledger entry behind even when its MRF intent is accepted
|
||||
//! (backlog#1894 axis A), so a lost intent is retried by the ledger rather
|
||||
//! than waiting for the failed-object TTL to re-scan the path.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
|
||||
@@ -186,11 +185,6 @@ impl MrfQueue {
|
||||
MrfQueuePushResult::Enqueued
|
||||
}
|
||||
|
||||
fn raise_limits_for_replay(&mut self, intents: usize, bytes: usize) {
|
||||
self.capacity = self.capacity.max(self.pending.len().saturating_add(intents));
|
||||
self.byte_budget = self.byte_budget.max(self.bytes.saturating_add(bytes));
|
||||
}
|
||||
|
||||
/// Bool compatibility adapter: only a newly executable queue item is
|
||||
/// reported as accepted; a coalesced duplicate is not durable admission.
|
||||
#[cfg(test)]
|
||||
@@ -591,8 +585,8 @@ impl MrfRuntime {
|
||||
self.dirty = true;
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot. This is not a durable successor receipt and
|
||||
// does not discharge the producer's existing retry hints.
|
||||
// smaller snapshot. The scanner ledger is cleared later, when the
|
||||
// canonical heal task reaches a successful terminal completion.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
@@ -660,10 +654,9 @@ pub fn spawn_mrf_consumer(manager: Arc<HealManager>) {
|
||||
|
||||
/// Replay the durable journal into a fresh pending queue and submit whatever
|
||||
/// it armed. Returns the number of intact intents replayed. Duplicates are
|
||||
/// merged by the manager's dedup key; the journal is retained whenever replay
|
||||
/// cannot fully hand off a successor in-memory snapshot (torn tails truncate
|
||||
/// via the per-record CRC). Public for integration tests; the live consumer
|
||||
/// invokes this through [`replay_into`] at startup.
|
||||
/// merged by the manager's dedup key; the journal file is removed once read
|
||||
/// (torn tails truncate via the per-record CRC). Public for integration tests;
|
||||
/// the live consumer invokes this through [`replay_into`] at startup.
|
||||
pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
@@ -676,13 +669,7 @@ struct ReplayOutcome {
|
||||
journal_on_disk: bool,
|
||||
}
|
||||
|
||||
fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize) -> bool {
|
||||
rearm_incomplete || pending_depth > 0
|
||||
}
|
||||
|
||||
/// Shared replay core: read + decode + re-arm, then drain what fits. The
|
||||
/// startup journal is removed only after every replayed record has either
|
||||
/// reached the manager or been proven redundant inside the in-memory queue.
|
||||
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
|
||||
async fn replay_into(
|
||||
manager: &Arc<HealManager>,
|
||||
queue: &mut MrfQueue,
|
||||
@@ -715,26 +702,13 @@ async fn replay_into(
|
||||
}
|
||||
counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX));
|
||||
let replayed = intents.len();
|
||||
let replay_bytes = intents
|
||||
.iter()
|
||||
.fold(0usize, |total, intent| total.saturating_add(intent.estimated_bytes()));
|
||||
// The decoded journal is already resident in memory. Allow the startup
|
||||
// queue to arm that full bounded snapshot so a later flush can become the
|
||||
// successor anchor instead of overwriting the old journal with only a
|
||||
// prefix.
|
||||
queue.raise_limits_for_replay(intents.len(), replay_bytes);
|
||||
let mut rearm_incomplete = false;
|
||||
for intent in intents {
|
||||
let result = queue.try_push_typed(intent.clone());
|
||||
match result {
|
||||
MrfQueuePushResult::Enqueued => {}
|
||||
MrfQueuePushResult::Coalesced => rustfs_common::mrf_channel::release_mrf_intent(&intent),
|
||||
MrfQueuePushResult::Rejected => {
|
||||
rearm_incomplete = true;
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
if !matches!(result, MrfQueuePushResult::Enqueued) {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
}
|
||||
let journal_on_disk = !delete_journals().await;
|
||||
|
||||
// Drain the replayed intents immediately; whatever the manager refuses
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
@@ -754,11 +728,6 @@ async fn replay_into(
|
||||
}
|
||||
}
|
||||
}
|
||||
let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth()) {
|
||||
true
|
||||
} else {
|
||||
!delete_journals().await
|
||||
};
|
||||
ReplayOutcome {
|
||||
replayed,
|
||||
journal_on_disk,
|
||||
@@ -778,13 +747,13 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
backoff_until: None,
|
||||
};
|
||||
|
||||
// Replay reads the journal and re-arms intents. The startup journal stays
|
||||
// on disk whenever any replayed intent still needs a successor snapshot.
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
runtime.journal_on_disk = replay.journal_on_disk;
|
||||
// Anything still pending (e.g. the manager was full and backoff armed)
|
||||
// must be re-persisted by the next flush before replay can delete the
|
||||
// startup anchor.
|
||||
// The replay deleted the journal file; anything still pending (e.g. the
|
||||
// manager was full and backoff armed) must be re-persisted by the next
|
||||
// flush or a crash before it would lose those intents.
|
||||
runtime.dirty = runtime.queue.depth() > 0;
|
||||
|
||||
let mut flush_tick = tokio::time::interval(runtime.config.flush_interval);
|
||||
@@ -920,38 +889,6 @@ mod tests {
|
||||
assert!(matches!(tick_action(false, 0, false), Idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() {
|
||||
assert!(
|
||||
replay_must_retain_journal(true, 0),
|
||||
"a rejected replay record still needs its disk anchor"
|
||||
);
|
||||
assert!(
|
||||
replay_must_retain_journal(false, 1),
|
||||
"a Full admission retry must keep the startup journal until the next snapshot"
|
||||
);
|
||||
assert!(
|
||||
!replay_must_retain_journal(false, 0),
|
||||
"only a fully consumed replay snapshot may be deleted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_can_arm_more_records_than_live_queue_budget() {
|
||||
let mut queue = MrfQueue::new(1, intent("bucket", "object-0", 0).estimated_bytes());
|
||||
let intents = vec![intent("bucket", "object-0", 0), intent("bucket", "object-1", 0)];
|
||||
let bytes = intents
|
||||
.iter()
|
||||
.fold(0usize, |total, intent| total.saturating_add(intent.estimated_bytes()));
|
||||
|
||||
queue.raise_limits_for_replay(intents.len(), bytes);
|
||||
|
||||
for intent in intents {
|
||||
assert_eq!(queue.try_push_typed(intent), MrfQueuePushResult::Enqueued);
|
||||
}
|
||||
assert_eq!(queue.depth(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
|
||||
@@ -123,13 +123,6 @@ pub struct CommittedSnapshot {
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SnapshotReadStats {
|
||||
file_reads: usize,
|
||||
bytes_read: usize,
|
||||
peak_file_bytes: usize,
|
||||
}
|
||||
|
||||
impl CommittedSnapshot {
|
||||
/// Persistent single-writer sequence, not a process UUID ordering.
|
||||
pub fn sequence(&self) -> u64 {
|
||||
@@ -169,15 +162,6 @@ pub enum RecoverySnapshot {
|
||||
}
|
||||
|
||||
async fn read_bounded(disk: &EcstoreDiskStore, path: &str, limit: usize) -> Result<Option<Vec<u8>>, SnapshotError> {
|
||||
read_bounded_with_stats(disk, path, limit, None).await
|
||||
}
|
||||
|
||||
async fn read_bounded_with_stats(
|
||||
disk: &EcstoreDiskStore,
|
||||
path: &str,
|
||||
limit: usize,
|
||||
mut stats: Option<&mut SnapshotReadStats>,
|
||||
) -> Result<Option<Vec<u8>>, SnapshotError> {
|
||||
let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(reader) => reader,
|
||||
Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) => return Ok(None),
|
||||
@@ -194,11 +178,6 @@ async fn read_bounded_with_stats(
|
||||
if bytes.len() > limit {
|
||||
return Err(SnapshotError::TooLarge);
|
||||
}
|
||||
if let Some(stats) = stats.as_mut() {
|
||||
stats.file_reads += 1;
|
||||
stats.bytes_read = stats.bytes_read.checked_add(bytes.len()).ok_or(SnapshotError::TooLarge)?;
|
||||
stats.peak_file_bytes = stats.peak_file_bytes.max(bytes.len());
|
||||
}
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
|
||||
@@ -218,26 +197,17 @@ fn select_snapshot(selected: &mut Option<CommittedSnapshot>, candidate: Committe
|
||||
}
|
||||
|
||||
async fn read_committed(disks: &[EcstoreDiskStore], limit: usize) -> Result<Option<CommittedSnapshot>, SnapshotError> {
|
||||
read_committed_with_stats(disks, limit, None).await
|
||||
}
|
||||
|
||||
async fn read_committed_with_stats(
|
||||
disks: &[EcstoreDiskStore],
|
||||
limit: usize,
|
||||
mut stats: Option<&mut SnapshotReadStats>,
|
||||
) -> Result<Option<CommittedSnapshot>, SnapshotError> {
|
||||
let mut selected = None;
|
||||
let mut damaged = None;
|
||||
let mut identities = HashMap::new();
|
||||
for disk in disks {
|
||||
for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) {
|
||||
let candidate = async {
|
||||
let Some(manifest) = read_bounded_with_stats(disk, manifest_path, MANIFEST_LEN, stats.as_deref_mut()).await?
|
||||
else {
|
||||
let Some(manifest) = read_bounded(disk, manifest_path, MANIFEST_LEN).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let header = Manifest::decode(&manifest, limit)?;
|
||||
let payload = read_bounded_with_stats(disk, payload_path, header.payload_len, stats.as_deref_mut())
|
||||
let payload = read_bounded(disk, payload_path, header.payload_len)
|
||||
.await?
|
||||
.ok_or(SnapshotError::Corrupt)?;
|
||||
CommittedSnapshot::decode(&manifest, payload, limit).map(Some)
|
||||
@@ -522,69 +492,6 @@ mod tests {
|
||||
assert_eq!(recovered.manifest.sequence, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_reader_reopens_previous_anchor_across_publication_boundaries() {
|
||||
let owner = Uuid::new_v4();
|
||||
let old = payload("old");
|
||||
let next = payload("next");
|
||||
let boundaries = [
|
||||
("payload-only", next.clone(), None),
|
||||
("torn-manifest", next.clone(), Some(manifest(owner, 2, &next)[..20].to_vec())),
|
||||
("stale-payload", old.clone(), Some(manifest(owner, 2, &next))),
|
||||
];
|
||||
for (case, successor_payload, successor_manifest) in boundaries {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let store = disk(&root, "disk").await;
|
||||
commit(&store, 0, owner, 1, &old).await;
|
||||
install(&store, PAYLOAD_PATHS[1], &successor_payload).await;
|
||||
if let Some(manifest) = &successor_manifest {
|
||||
install(&store, MANIFEST_PATHS[1], manifest).await;
|
||||
}
|
||||
|
||||
let reopened = disk(&root, "disk").await;
|
||||
let recovered = read_committed(std::slice::from_ref(&reopened), 4096)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{case}: old anchor must remain readable after reopen: {error:?}"))
|
||||
.unwrap_or_else(|| panic!("{case}: previous committed anchor missing after reopen"));
|
||||
assert_eq!(recovered.manifest.sequence, 1, "{case}: successor must not become authoritative");
|
||||
assert_eq!(recovered.payload, old, "{case}: previous payload must survive");
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0])
|
||||
.await
|
||||
.expect("old payload retained")
|
||||
.as_ref(),
|
||||
old.as_slice(),
|
||||
"{case}: previous payload bytes changed"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0])
|
||||
.await
|
||||
.expect("old manifest retained")
|
||||
.as_ref(),
|
||||
manifest(owner, 1, &old).as_slice(),
|
||||
"{case}: previous manifest bytes changed"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1])
|
||||
.await
|
||||
.expect("successor payload retained")
|
||||
.as_ref(),
|
||||
successor_payload.as_slice(),
|
||||
"{case}: successor evidence changed"
|
||||
);
|
||||
if let Some(manifest) = &successor_manifest {
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1])
|
||||
.await
|
||||
.expect("successor manifest retained")
|
||||
.as_ref(),
|
||||
manifest.as_slice(),
|
||||
"{case}: successor manifest evidence changed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_manifest_cas_cannot_replace_committed_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
@@ -664,35 +571,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_reader_resource_bounds_are_measured() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let owner = Uuid::new_v4();
|
||||
let old = [payload("old-0"), payload("old-1")].concat();
|
||||
let new = [payload("new-0"), payload("new-1"), payload("new-2")].concat();
|
||||
commit(&first, 0, owner, 1, &old).await;
|
||||
commit(&second, 1, owner, 2, &new).await;
|
||||
|
||||
let mut stats = SnapshotReadStats::default();
|
||||
let recovered = read_committed_with_stats(&[first, second], 4096, Some(&mut stats))
|
||||
.await
|
||||
.expect("read committed replicas")
|
||||
.expect("committed snapshot");
|
||||
|
||||
assert_eq!(recovered.sequence(), 2);
|
||||
assert_eq!(recovered.payload(), new.as_slice());
|
||||
assert_eq!(recovered.manifest.payload_len, new.len());
|
||||
assert_eq!(stats.file_reads, 4, "only committed manifests and their payloads are materialized");
|
||||
assert_eq!(stats.bytes_read, (MANIFEST_LEN * 2) + old.len() + new.len());
|
||||
assert_eq!(
|
||||
stats.peak_file_bytes,
|
||||
new.len().max(MANIFEST_LEN),
|
||||
"reader peak allocation remains bounded by one manifest or payload file"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_inspection_rejects_complete_subsets_and_scope_ambiguity() {
|
||||
let scoped = |set_index| {
|
||||
|
||||
+11
-179
@@ -15,7 +15,6 @@
|
||||
//! Execution results are separate from repair responsibility. A legacy
|
||||
//! successful storage call supplies no authoritative repair receipt.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::VecDeque, time::SystemTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -23,16 +22,14 @@ const MAX_OUTCOME_ITEMS: usize = 128;
|
||||
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
|
||||
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealObjectKind {
|
||||
Object,
|
||||
Metadata,
|
||||
Decode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectIdentity {
|
||||
pub kind: HealObjectKind,
|
||||
pub bucket: String,
|
||||
@@ -44,8 +41,7 @@ pub struct HealObjectIdentity {
|
||||
pub set_index: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealDeferredReason {
|
||||
DanglingDeleteGrace,
|
||||
TransientUsageCache,
|
||||
@@ -53,21 +49,14 @@ pub enum HealDeferredReason {
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealFailureClass {
|
||||
Recoverable,
|
||||
RetryExhausted,
|
||||
Permanent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(
|
||||
tag = "state",
|
||||
content = "details",
|
||||
rename_all = "snake_case",
|
||||
rename_all_fields = "camelCase"
|
||||
)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HealObjectDisposition {
|
||||
/// The legacy storage response does not prove the requested check or commit.
|
||||
Unknown,
|
||||
@@ -83,8 +72,7 @@ pub enum HealObjectDisposition {
|
||||
DryRunObserved,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectOutcome {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
@@ -101,8 +89,7 @@ impl HealObjectOutcome {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealTraversalCoverage {
|
||||
#[default]
|
||||
Unknown,
|
||||
@@ -110,16 +97,14 @@ pub enum HealTraversalCoverage {
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealAbortReason {
|
||||
Cancelled,
|
||||
Deadline,
|
||||
Untraversable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "state", content = "reason", rename_all = "snake_case")]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealExecutionOutcome {
|
||||
#[default]
|
||||
Pending,
|
||||
@@ -129,8 +114,7 @@ pub enum HealExecutionOutcome {
|
||||
Aborted(HealAbortReason),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealOutcomeCounters {
|
||||
pub processed: u64,
|
||||
pub healed: u64,
|
||||
@@ -143,8 +127,7 @@ pub struct HealOutcomeCounters {
|
||||
pub overflowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealTaskOutcome {
|
||||
pub execution: HealExecutionOutcome,
|
||||
pub coverage: HealTraversalCoverage,
|
||||
@@ -152,99 +135,11 @@ pub struct HealTaskOutcome {
|
||||
/// A bounded diagnostic window, not a complete responsibility ledger.
|
||||
pub objects: VecDeque<HealObjectOutcome>,
|
||||
pub objects_truncated: bool,
|
||||
#[serde(skip)]
|
||||
retained_object_bytes: usize,
|
||||
#[serde(skip)]
|
||||
untraversable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum HealOutcomeWireError {
|
||||
#[error("heal outcome is missing execution or counters")]
|
||||
MissingFields,
|
||||
#[error("heal outcome has invalid or unsupported execution fields")]
|
||||
InvalidFields(#[from] serde_json::Error),
|
||||
#[error("finished heal summary contradicts its canonical outcome")]
|
||||
ContradictoryCompletion,
|
||||
}
|
||||
|
||||
/// Reconcile a peer's successful legacy summary using the canonical owner types.
|
||||
/// A running retry may legitimately retain the preceding attempt's outcome.
|
||||
pub fn legacy_wire_status<'a>(
|
||||
summary: &'a str,
|
||||
wire: &serde_json::Value,
|
||||
truncated: bool,
|
||||
) -> Result<(&'a str, Option<String>), HealOutcomeWireError> {
|
||||
if summary != "finished" {
|
||||
return Ok((summary, None));
|
||||
}
|
||||
let execution = HealExecutionOutcome::deserialize(wire.get("execution").ok_or(HealOutcomeWireError::MissingFields)?)?;
|
||||
let counters = HealOutcomeCounters::deserialize(wire.get("counters").ok_or(HealOutcomeWireError::MissingFields)?)?;
|
||||
if matches!(execution, HealExecutionOutcome::Pending | HealExecutionOutcome::Running)
|
||||
|| (execution == HealExecutionOutcome::Completed && counters.failed > 0)
|
||||
{
|
||||
return Err(HealOutcomeWireError::ContradictoryCompletion);
|
||||
}
|
||||
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
|
||||
Ok((
|
||||
adapted,
|
||||
if adapted != summary {
|
||||
heal_status_detail(detail, truncated)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
|
||||
if !truncated {
|
||||
return detail;
|
||||
}
|
||||
let truncation = "heal result items were truncated";
|
||||
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
|
||||
}
|
||||
|
||||
fn legacy_execution_status<'a>(
|
||||
summary: &'a str,
|
||||
detail: Option<String>,
|
||||
execution: HealExecutionOutcome,
|
||||
counters: &HealOutcomeCounters,
|
||||
) -> (&'a str, Option<String>) {
|
||||
if summary != "finished" {
|
||||
return (summary, detail);
|
||||
}
|
||||
match execution {
|
||||
HealExecutionOutcome::CompletedWithErrors => (
|
||||
"stopped",
|
||||
Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
|
||||
),
|
||||
HealExecutionOutcome::Aborted(reason) => {
|
||||
let reason = match reason {
|
||||
HealAbortReason::Cancelled => "cancelled",
|
||||
HealAbortReason::Deadline => "timed out",
|
||||
HealAbortReason::Untraversable => "untraversable",
|
||||
};
|
||||
("stopped", Some(format!("heal task {reason}")))
|
||||
}
|
||||
HealExecutionOutcome::Completed if counters.unknown > 0 => (
|
||||
summary,
|
||||
Some(format!(
|
||||
"heal traversal completed; authoritative storage proof is unavailable for {} objects",
|
||||
counters.unknown
|
||||
)),
|
||||
),
|
||||
HealExecutionOutcome::Pending | HealExecutionOutcome::Running => {
|
||||
("running", Some("heal execution has not reached a terminal outcome".to_string()))
|
||||
}
|
||||
HealExecutionOutcome::Completed => (summary, detail),
|
||||
}
|
||||
}
|
||||
|
||||
impl HealTaskOutcome {
|
||||
pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
|
||||
legacy_execution_status(summary, detail, self.execution, &self.counters)
|
||||
}
|
||||
|
||||
pub(crate) fn start(&mut self) {
|
||||
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
self.execution = HealExecutionOutcome::Running;
|
||||
@@ -397,69 +292,6 @@ mod canonical_outcome_tests {
|
||||
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_serialization_keeps_unverified_dispositions_and_window_bounds() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for disposition in [
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
},
|
||||
HealObjectDisposition::DryRunObserved,
|
||||
] {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
outcome.finish(None);
|
||||
let wire = serde_json::to_value(&outcome).expect("canonical wire view");
|
||||
assert_eq!(wire["execution"]["state"], "completed");
|
||||
assert_eq!(wire["counters"]["healed"], 0);
|
||||
assert_eq!(wire["counters"]["skipped"], 3);
|
||||
assert_eq!(wire["objects"][1]["disposition"]["details"]["reason"], "dangling_delete_grace");
|
||||
assert!(wire["objects"][1]["identity"]["bucketIncarnationId"].is_null());
|
||||
for _ in 0..MAX_OUTCOME_ITEMS + 1 {
|
||||
let mut result = item(HealObjectDisposition::Unknown);
|
||||
result.detail = Some("\"".repeat(MAX_OUTCOME_DETAIL_BYTES));
|
||||
outcome.record(result);
|
||||
}
|
||||
let bytes = serde_json::to_vec(&outcome).expect("bounded canonical samples");
|
||||
assert!(
|
||||
bytes.len() < 8 * MAX_OUTCOME_BYTES,
|
||||
"JSON escaping remains bounded independently of object count"
|
||||
);
|
||||
assert!(outcome.objects_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_wire_consistency_rejects_unknown_success_without_rejecting_extensions() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.finish(None);
|
||||
let mut wire = serde_json::to_value(&outcome).expect("canonical snapshot");
|
||||
wire["execution"]["futureField"] = serde_json::json!({"new": true});
|
||||
wire["counters"]["futureCounter"] = serde_json::json!(42);
|
||||
assert_eq!(
|
||||
legacy_wire_status("finished", &wire, false).expect("unknown extension fields"),
|
||||
("finished", None)
|
||||
);
|
||||
wire["execution"]["state"] = serde_json::json!("future_execution");
|
||||
assert!(legacy_wire_status("finished", &wire, false).is_err());
|
||||
assert_eq!(
|
||||
legacy_wire_status("running", &wire, false).expect("unknown nonterminal outcome"),
|
||||
("running", None)
|
||||
);
|
||||
wire["execution"] = serde_json::json!({"state":"completed"});
|
||||
wire["counters"]["failed"] = serde_json::json!(1);
|
||||
assert!(matches!(
|
||||
legacy_wire_status("finished", &wire, false),
|
||||
Err(HealOutcomeWireError::ContradictoryCompletion)
|
||||
));
|
||||
wire.as_object_mut().expect("object").remove("execution");
|
||||
assert!(matches!(
|
||||
legacy_wire_status("finished", &wire, false),
|
||||
Err(HealOutcomeWireError::MissingFields)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
|
||||
@@ -18,25 +18,6 @@ use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
};
|
||||
|
||||
/// Whether automatic replacement may fall back to the set-wide format heal when
|
||||
/// a target cannot pass the independent-mount admission.
|
||||
///
|
||||
/// Directory-backed deployments already declare, through
|
||||
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
|
||||
/// directories sharing a device with the host root. Those endpoints can never
|
||||
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
|
||||
/// runtime-wiped or replaced directory disk would stay deferred forever. The
|
||||
/// admission check itself is never bypassed; the fallback only routes the heal
|
||||
/// through the ordinary format path that formats every unformatted disk in the
|
||||
/// set, which is exactly what the pre-admission `heal_disk` path did.
|
||||
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool_with_aliases(
|
||||
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||
&[rustfs_config::ENV_MINIO_CI],
|
||||
rustfs_config::DEFAULT_UNSAFE_BYPASS_DISK_CHECK,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
}
|
||||
@@ -203,47 +184,12 @@ mod tests {
|
||||
assert!(endpoint.is_local);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_backed_fallback_is_off_by_default() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
|| assert!(!directory_backed_replacement_fallback_enabled()),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_backed_fallback_follows_the_disk_check_bypass() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
|| assert!(directory_backed_replacement_fallback_enabled()),
|
||||
);
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("false")),
|
||||
(rustfs_config::ENV_MINIO_CI, Some("true")),
|
||||
],
|
||||
|| {
|
||||
assert!(
|
||||
!directory_backed_replacement_fallback_enabled(),
|
||||
"the canonical key must win over the alias"
|
||||
)
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_environment_cannot_bypass_mount_admission() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_TEST_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")),
|
||||
("RUSTFS_E2E_AUTO_REPLACEMENT_READINESS_BYPASS", Some("1")),
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
],
|
||||
async {
|
||||
let temp = TempDir::new().expect("temporary replacement root should be created");
|
||||
|
||||
@@ -14,107 +14,8 @@
|
||||
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
|
||||
use super::*;
|
||||
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
|
||||
use crate::heal::storage::HealListItem;
|
||||
use crate::heal::utils::format_set_disk_id;
|
||||
|
||||
const MAX_DEFERRED_OBJECTS: usize = 256;
|
||||
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
|
||||
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
|
||||
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
|
||||
|
||||
struct DeferredObject {
|
||||
name: String,
|
||||
version_id: Option<String>,
|
||||
attempt: u32,
|
||||
page: u64,
|
||||
first_failure: Option<tokio::time::Instant>,
|
||||
due: tokio::time::Instant,
|
||||
}
|
||||
|
||||
impl DeferredObject {
|
||||
fn new(item: HealListItem, page: u64) -> Self {
|
||||
Self {
|
||||
name: item.name,
|
||||
version_id: item.version_id,
|
||||
attempt: 0,
|
||||
page,
|
||||
first_failure: None,
|
||||
due: tokio::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn payload_bytes(&self) -> usize {
|
||||
self.name
|
||||
.capacity()
|
||||
.saturating_add(self.version_id.as_ref().map_or(0, String::capacity))
|
||||
}
|
||||
|
||||
fn expired(&self) -> bool {
|
||||
self.first_failure.is_some_and(|first| first.elapsed() >= MAX_DEFERRED_AGE)
|
||||
}
|
||||
|
||||
fn defer(&mut self, delay: Duration) {
|
||||
let now = tokio::time::Instant::now();
|
||||
let first = *self.first_failure.get_or_insert(now);
|
||||
self.attempt += 1;
|
||||
self.due = (now + delay).min(first + MAX_DEFERRED_AGE);
|
||||
}
|
||||
}
|
||||
|
||||
// Only failed identities are retained. The current listing page remains owned
|
||||
// by the caller; capacity pressure stops fetching, never discards that page.
|
||||
struct DeferredWindow {
|
||||
objects: VecDeque<DeferredObject>,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for DeferredWindow {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
objects: VecDeque::new(),
|
||||
// Charge every possible slot up front, including spare capacity.
|
||||
bytes: MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DeferredWindow {
|
||||
fn push(&mut self, item: DeferredObject) -> std::result::Result<(), DeferredObject> {
|
||||
let bytes = item.payload_bytes();
|
||||
if self.objects.len() >= MAX_DEFERRED_OBJECTS || bytes > MAX_DEFERRED_BYTES.saturating_sub(self.bytes) {
|
||||
return Err(item);
|
||||
}
|
||||
self.bytes += bytes;
|
||||
self.objects.push_back(item);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pop_due(&mut self) -> Option<DeferredObject> {
|
||||
let now = tokio::time::Instant::now();
|
||||
let index = self.objects.iter().position(|item| item.due <= now)?;
|
||||
let item = self.objects.remove(index)?;
|
||||
self.bytes -= item.payload_bytes();
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn next_due(&self) -> Option<tokio::time::Instant> {
|
||||
self.objects.iter().map(|item| item.due).min()
|
||||
}
|
||||
|
||||
fn can_advance(&self, page: u64) -> bool {
|
||||
self.objects.len() < MAX_DEFERRED_OBJECTS
|
||||
&& self.bytes < MAX_DEFERRED_BYTES
|
||||
&& self
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| page.saturating_sub(item.page) < MAX_DEFERRED_FORWARD_PAGES)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/deferred_retry_window.rs"]
|
||||
mod deferred_retry_window;
|
||||
|
||||
fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Option<Error> {
|
||||
if opts.dry_run || !opts.recreate {
|
||||
return None;
|
||||
@@ -404,122 +305,83 @@ impl HealTask {
|
||||
|
||||
for (set_disk_id, heal_opts) in listing_scopes {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut deferred = DeferredWindow::default();
|
||||
let mut inline_retry: Option<DeferredObject> = None;
|
||||
let mut page_number = 0_u64;
|
||||
let mut aborted_progress_unknown = false;
|
||||
let mut pending = Vec::<HealListItem>::new().into_iter();
|
||||
let mut listing_finished = false;
|
||||
let mut listing_attempt = 0;
|
||||
let mut listing_due = tokio::time::Instant::now();
|
||||
let scope_result: Result<()> = async {
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none()
|
||||
{
|
||||
break;
|
||||
}
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let mut listing_attempt = 0;
|
||||
let (objects, next_token, is_truncated) = loop {
|
||||
self.pace_mainline().await?;
|
||||
// Listing and object retries share this safe boundary. A
|
||||
// failed listing never hides an already-due object retry.
|
||||
let item = deferred.pop_due().or_else(|| {
|
||||
if inline_retry
|
||||
.as_ref()
|
||||
.is_some_and(|item| item.due <= tokio::time::Instant::now())
|
||||
{
|
||||
inline_retry.take()
|
||||
} else if inline_retry.is_none() {
|
||||
pending.next().map(|item| DeferredObject::new(item, page_number))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
let Some(mut item) = item else {
|
||||
let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
|
||||
if can_list && listing_due <= tokio::time::Instant::now() {
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok((objects, next_token, is_truncated)) => {
|
||||
page_number = page_number.saturating_add(1);
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
listing_finished = continuation_token.is_none();
|
||||
listing_attempt = 0;
|
||||
listing_due = tokio::time::Instant::now();
|
||||
pending = objects.into_iter();
|
||||
}
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
listing_due =
|
||||
tokio::time::Instant::now() + self.bucket_object_retry_delay(listing_attempt);
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
let due = deferred
|
||||
.next_due()
|
||||
.into_iter()
|
||||
.chain(inline_retry.as_ref().map(|item| item.due))
|
||||
.chain(can_list.then_some(listing_due))
|
||||
.min();
|
||||
if let Some(due) = due {
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok(page) => break page,
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep_until(due).await;
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
};
|
||||
let retry_attempt = item.attempt;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
}
|
||||
};
|
||||
|
||||
let mut terminal_outcome = true;
|
||||
let age_exhausted = item.expired();
|
||||
let error = if age_exhausted {
|
||||
Some(Error::other("heal object retry age exhausted"))
|
||||
} else {
|
||||
match self
|
||||
let mut pending = objects;
|
||||
let mut retry_attempt = 0_u32;
|
||||
while !pending.is_empty() {
|
||||
if retry_attempt > 0 {
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let mut retry = Vec::with_capacity(pending.len());
|
||||
for item in pending {
|
||||
self.check_control_flags().await?;
|
||||
self.pace_mainline().await?;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
}
|
||||
|
||||
let mut terminal_outcome = true;
|
||||
let error = match self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
@@ -552,103 +414,35 @@ impl HealTask {
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
return Err(err);
|
||||
}
|
||||
_ => self.outcome.write().await.attempt_failed(),
|
||||
}
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
return Err(err);
|
||||
}
|
||||
_ if !age_exhausted => self.outcome.write().await.attempt_failed(),
|
||||
_ => {}
|
||||
}
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "dangling_delete_grace_skip",
|
||||
error = %err,
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if !age_exhausted && err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
item.defer(self.bucket_object_retry_delay(retry_attempt + 1));
|
||||
if let Err(item) = deferred.push(item) {
|
||||
inline_retry = Some(item);
|
||||
}
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if age_exhausted || err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
@@ -657,83 +451,115 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
result = "dangling_delete_grace_skip",
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if terminal_outcome {
|
||||
telemetry_unknown |= !increment_counter(&mut scanned);
|
||||
}
|
||||
if terminal_outcome {
|
||||
telemetry_unknown |= !increment_counter(&mut scanned);
|
||||
}
|
||||
|
||||
if !terminal_outcome {
|
||||
continue;
|
||||
}
|
||||
if !terminal_outcome {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
}
|
||||
pending = retry;
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = scope_result {
|
||||
let disposition = match error {
|
||||
Error::TaskCancelled => HealObjectDisposition::Cancelled,
|
||||
Error::TaskTimeout => HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
},
|
||||
_ => HealObjectDisposition::Unknown,
|
||||
};
|
||||
// Only attempted identities have terminal outcomes. Unstarted
|
||||
// page tails remain unprocessed under the task's partial coverage.
|
||||
// No detached sleepers survive abort.
|
||||
for item in deferred.objects.into_iter().chain(inline_retry) {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity: self.outcome_identity(
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
heal_opts.pool,
|
||||
heal_opts.set,
|
||||
),
|
||||
disposition: disposition.clone(),
|
||||
detail: None,
|
||||
});
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if aborted_progress_unknown {
|
||||
progress.mark_unknown();
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated without a continuation token is a compatibility EOF.
|
||||
break;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,36 +42,7 @@ impl HealTask {
|
||||
progress.update_stage(0, 4);
|
||||
}
|
||||
|
||||
let mut is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
if is_auto_replacement
|
||||
&& crate::heal::replacement_readiness::directory_backed_replacement_fallback_enabled()
|
||||
&& self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
// Directory-backed endpoints cannot pass replacement admission; the
|
||||
// operator opted out of disk checks, so heal the set the way the
|
||||
// pre-admission `heal_disk` path did instead of deferring forever.
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "replacement_admission",
|
||||
result = "directory_backed_fallback",
|
||||
target_count = self.heal_endpoints.len(),
|
||||
"Heal erasure set falls back to set-wide format heal for a replacement target that is not an independently mounted disk"
|
||||
);
|
||||
is_auto_replacement = false;
|
||||
}
|
||||
let replacement_targets = if is_auto_replacement {
|
||||
self.heal_endpoints.clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
let mut requested_targets = self.heal_endpoints.clone();
|
||||
requested_targets.sort_unstable();
|
||||
@@ -450,7 +421,7 @@ impl HealTask {
|
||||
heal_opts,
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone())
|
||||
.with_mainline_pacer(self.mainline_pacer.clone());
|
||||
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
|
||||
mod deferred_retry;
|
||||
|
||||
mod canonical_outcome {
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
@@ -480,95 +478,6 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
);
|
||||
}
|
||||
|
||||
fn directory_backed_replacement_request() -> HealRequest {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = vec!["/data/disk0".to_string()];
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_backed_replacement_falls_back_to_set_format_when_disk_checks_are_bypassed() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
async {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(false),
|
||||
global_format_ok_endpoints: Mutex::new(vec!["/data/disk0".to_string()]),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(directory_backed_replacement_request(), storage.clone());
|
||||
|
||||
// The mock has no local disk behind "/data/disk0", so the run stops at
|
||||
// the healing-marker step that follows the format stage, exactly like
|
||||
// `automatic_replacement_uses_target_scoped_format`. The assertions
|
||||
// below pin which format path ran before that point.
|
||||
let err = task.execute().await.expect_err("the mock has no local healing marker target");
|
||||
assert!(
|
||||
err.to_string().contains("healing marker target is unavailable"),
|
||||
"the fallback must reach the post-format marker step, got: {err}"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
*storage.global_format_calls.lock().unwrap(),
|
||||
1,
|
||||
"the fallback must run exactly one set-wide format heal"
|
||||
);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"the fallback must not run the target-scoped replacement format"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_backed_replacement_stays_fail_closed_without_disk_check_bypass() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>),
|
||||
(rustfs_config::ENV_MINIO_CI, None::<&str>),
|
||||
],
|
||||
async {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(false),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(directory_backed_replacement_request(), storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect_err("an inadmissible replacement target must keep failing closed");
|
||||
|
||||
assert_eq!(
|
||||
*storage.global_format_calls.lock().unwrap(),
|
||||
0,
|
||||
"fail-closed admission must not format the set"
|
||||
);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"fail-closed admission must not format the target"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_persists_intent_before_format() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -1030,10 +939,6 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
retry_test_pages: Option<Vec<Vec<HealListItem>>>,
|
||||
retry_test_delays: HashMap<String, Duration>,
|
||||
retry_test_listing_delays: Mutex<VecDeque<Duration>>,
|
||||
retry_test_events: Mutex<Vec<String>>,
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
fail_second_listing_page: bool,
|
||||
@@ -1051,8 +956,6 @@ struct MockStorage {
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
/// Endpoints the set-wide format mock reports as freshly formatted (`state == "ok"`).
|
||||
global_format_ok_endpoints: Mutex<Vec<String>>,
|
||||
replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
|
||||
replacement_target_identities_ready: Mutex<bool>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<crate::heal::resume::ReplacementTargetIdentity>>>,
|
||||
@@ -1206,7 +1109,6 @@ fn replacement_identity(
|
||||
}
|
||||
|
||||
enum MockHealObjectOutcome {
|
||||
RetryableLock,
|
||||
OkWithOtherError(&'static str),
|
||||
ErrOther(&'static str),
|
||||
DanglingGraceDeferred,
|
||||
@@ -1311,10 +1213,6 @@ impl HealStorageAPI for MockStorage {
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.heal_object_calls.lock().unwrap().push(object.to_string());
|
||||
self.retry_test_events.lock().expect("events").push(format!("heal:{object}"));
|
||||
if let Some(delay) = self.retry_test_delays.get(object) {
|
||||
tokio::time::sleep(*delay).await;
|
||||
}
|
||||
self.heal_object_version_ids
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -1343,13 +1241,6 @@ impl HealStorageAPI for MockStorage {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableLock => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
|
||||
resource: object.to_string(),
|
||||
owner: "competing-writer".to_string(),
|
||||
}))),
|
||||
)),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
@@ -1375,13 +1266,6 @@ impl HealStorageAPI for MockStorage {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableLock => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
|
||||
resource: object.to_string(),
|
||||
owner: "competing-writer".to_string(),
|
||||
}))),
|
||||
)),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
@@ -1429,26 +1313,10 @@ impl HealStorageAPI for MockStorage {
|
||||
return Err(error);
|
||||
}
|
||||
let no_heal_required = *self.format_no_heal_required.lock().unwrap();
|
||||
let result = HealResultItem {
|
||||
after: Infos {
|
||||
drives: self
|
||||
.global_format_ok_endpoints
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|endpoint| HealDriveInfo {
|
||||
endpoint: endpoint.clone(),
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
if no_heal_required {
|
||||
Ok((result, Some(Error::Storage(EcstoreError::NoHealRequired))))
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
|
||||
} else {
|
||||
Ok((result, None))
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1493,19 +1361,6 @@ impl HealStorageAPI for MockStorage {
|
||||
.lock()
|
||||
.expect("listing tokens")
|
||||
.push(continuation_token.map(ToOwned::to_owned));
|
||||
self.retry_test_events
|
||||
.lock()
|
||||
.expect("events")
|
||||
.push(format!("list:{}", continuation_token.unwrap_or("first")));
|
||||
let delay = self.retry_test_listing_delays.lock().expect("listing delays").pop_front();
|
||||
if let Some(delay) = delay {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if let Some(pages) = &self.retry_test_pages {
|
||||
let page = continuation_token.map_or(0, |token| token.parse::<usize>().expect("test page token"));
|
||||
let next = (page + 1 < pages.len()).then(|| (page + 1).to_string());
|
||||
return Ok((pages[page].clone(), next.clone(), next.is_some()));
|
||||
}
|
||||
if let Some(remaining) = self
|
||||
.recoverable_second_page_failures
|
||||
.lock()
|
||||
|
||||
@@ -1,487 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
|
||||
HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
)
|
||||
}
|
||||
|
||||
fn pages_storage(pages: &[&[&str]]) -> MockStorage {
|
||||
MockStorage {
|
||||
retry_test_pages: Some(
|
||||
pages
|
||||
.iter()
|
||||
.map(|page| page.iter().map(|name| heal_item(name)).collect())
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn fail_once(storage: &MockStorage, name: &str) {
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), VecDeque::from([MockHealObjectOutcome::RetryableLock]));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])),
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([
|
||||
MockHealObjectOutcome::RetryableSlowDown,
|
||||
MockHealObjectOutcome::RetryableSlowDown,
|
||||
]),
|
||||
);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(30_500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "heal:object-a"]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
);
|
||||
execution.await.expect_err("age exhausted object must remain a batch failure");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
[
|
||||
"list:first",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-b"
|
||||
]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(31_500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second"]
|
||||
);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 1);
|
||||
execution.await.expect_err("age exhaustion remains a failure");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unprocessed() {
|
||||
for cancel in [true, false] {
|
||||
let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
|
||||
let mut page: Vec<HealListItem> = names.iter().map(|name| heal_item(name)).collect();
|
||||
page[256].version_id = Some("inline-version".to_string());
|
||||
let storage = Arc::new(MockStorage {
|
||||
retry_test_pages: Some(vec![page, vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
});
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
}
|
||||
let mut task = bucket_task(storage.clone());
|
||||
if !cancel {
|
||||
task.options.timeout = Some(Duration::from_secs(1));
|
||||
}
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
if cancel {
|
||||
task.cancel().await.expect("cancel");
|
||||
}
|
||||
let result = execution.await;
|
||||
assert!(matches!(
|
||||
(&result, cancel),
|
||||
(Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
|
||||
));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(
|
||||
outcome.counters.processed,
|
||||
outcome.counters.skipped,
|
||||
outcome.counters.failed,
|
||||
outcome.counters.healed
|
||||
),
|
||||
(257, 257, 0, 0)
|
||||
);
|
||||
assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
outcome.execution,
|
||||
crate::heal::outcome::HealExecutionOutcome::Aborted(if cancel {
|
||||
HealAbortReason::Cancelled
|
||||
} else {
|
||||
HealAbortReason::Deadline
|
||||
})
|
||||
);
|
||||
let inline: Vec<_> = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "blocked-256")
|
||||
.collect();
|
||||
assert_eq!(inline.len(), 1);
|
||||
assert_eq!(inline[0].identity.version_id.as_deref(), Some("inline-version"));
|
||||
assert_eq!(
|
||||
inline[0].disposition,
|
||||
if cancel {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
}
|
||||
);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(
|
||||
(
|
||||
progress.objects_scanned,
|
||||
progress.skipped_objects,
|
||||
progress.objects_failed,
|
||||
progress.objects_healed
|
||||
),
|
||||
(257, 257, 0, 0)
|
||||
);
|
||||
assert!(
|
||||
!outcome
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.identity.object == "blocked-257" || item.identity.object == "healthy")
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
}
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
|
||||
execution.await.expect_err("all four objects exhaust retries");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(4, 4, 16)
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn listing_retry_keeps_cursor_and_does_not_replay_successful_objects() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect("both retries complete");
|
||||
assert_eq!(
|
||||
storage.listing_tokens.lock().expect("tokens").as_slice(),
|
||||
[None, Some("second".to_string()), Some("second".to_string())]
|
||||
);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-a", "object-b"]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn typed_lock_contention_allows_only_two_forward_pages() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
fail_once(&storage, "a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c"]);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
|
||||
execution.await.expect("all objects complete");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c", "a", "d"]);
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() {
|
||||
let mut storage = pages_storage(&[&["a"], &["b", "c"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(3));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
bucket_task(storage.clone()).execute().await.expect("all objects complete");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "c"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
|
||||
let mut storage = pages_storage(&[&["a"], &["b"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("aged pending responsibility is not success");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(outcome.counters.failed, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a outcome")
|
||||
.disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_drains_owned_retries_once() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
|
||||
fail_once(&storage, "a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
task.cancel().await.expect("cancel");
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == "a").count(), 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a")
|
||||
.disposition,
|
||||
HealObjectDisposition::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn deadline_drains_owned_retries_without_false_completion() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
|
||||
fail_once(&storage, "a");
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.options.timeout = Some(Duration::from_secs(1));
|
||||
assert!(matches!(task.execute().await, Err(Error::TaskTimeout)));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a")
|
||||
.disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None
|
||||
}
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn full_window_backpressures_without_losing_the_current_page_tail() {
|
||||
let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
|
||||
let mut storage = MockStorage {
|
||||
retry_test_pages: Some(vec![names.iter().map(|name| heal_item(name)).collect(), vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
};
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
}
|
||||
// The last item has a version, proving the current-page tail is not rebuilt
|
||||
// from names alone when the window fills.
|
||||
storage.retry_test_pages.as_mut().expect("pages")[0][257].version_id = Some("version-tail".to_string());
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
execution.await.expect("every owned item eventually completes");
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 259);
|
||||
let calls = storage.heal_object_calls.lock().expect("calls");
|
||||
for name in &names {
|
||||
assert_eq!(calls.iter().filter(|called| *called == name).count(), 2);
|
||||
}
|
||||
let versions = storage.heal_object_version_ids.lock().expect("versions");
|
||||
for (name, version) in calls.iter().zip(versions.iter()) {
|
||||
if name == "blocked-257" {
|
||||
assert_eq!(version.as_deref(), Some("version-tail"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn oversized_identity_stays_inline_without_losing_version() {
|
||||
let name = "k".repeat(256 * 1024);
|
||||
let mut item = heal_item(&name);
|
||||
item.version_id = Some("v".repeat(1024));
|
||||
let storage = Arc::new(MockStorage {
|
||||
retry_test_pages: Some(vec![vec![item], vec![heal_item("healthy")]]),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, &name);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
|
||||
execution.await.expect("oversized identity retries inline");
|
||||
assert_eq!(task.get_outcome().await.counters.processed, 2);
|
||||
let versions = storage.heal_object_version_ids.lock().expect("versions");
|
||||
assert_eq!(versions[0], versions[1]);
|
||||
assert_eq!(versions[0].as_ref().expect("version").len(), 1024);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn terminal_listing_failure_keeps_deferred_identity_unknown() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
fail_second_listing_page: true,
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("listing cannot continue");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
|
||||
assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn healthy_second_page_advances_before_first_retry_is_due() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(0)),
|
||||
..Default::default()
|
||||
});
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableSlowDown]));
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err(),
|
||||
"the deferred first object must remain pending before its retry is due"
|
||||
);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-b"],
|
||||
"a retryable page head must not hold the healthy second page behind its backoff"
|
||||
);
|
||||
execution.await.expect("retry eventually succeeds");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["object-a", "object-b", "object-a"]
|
||||
);
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn item(name: String, version_id: Option<String>) -> DeferredObject {
|
||||
DeferredObject::new(
|
||||
HealListItem {
|
||||
name,
|
||||
version_id,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
},
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn count_cap_and_next_item_preserve_ownership() {
|
||||
let mut window = DeferredWindow::default();
|
||||
for _ in 0..MAX_DEFERRED_OBJECTS {
|
||||
assert!(window.push(item("key".to_string(), None)).is_ok());
|
||||
}
|
||||
let rejected = window
|
||||
.push(item("next".to_string(), Some("version".to_string())))
|
||||
.expect_err("count cap");
|
||||
assert_eq!(rejected.name, "next");
|
||||
assert_eq!(rejected.version_id.as_deref(), Some("version"));
|
||||
assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS);
|
||||
assert!(window.bytes <= MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.pop_due().is_some());
|
||||
assert!(window.push(rejected).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn byte_cap_counts_key_version_and_reserved_slots() {
|
||||
let mut window = DeferredWindow::default();
|
||||
let available = MAX_DEFERRED_BYTES - window.bytes;
|
||||
let key = "k".repeat(available / 2);
|
||||
let version = "v".repeat(available - key.capacity());
|
||||
assert_eq!(key.capacity() + version.capacity(), available);
|
||||
assert!(window.push(item(key, Some(version))).is_ok());
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.push(item("x".to_string(), None)).is_err());
|
||||
assert!(window.pop_due().is_some());
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>());
|
||||
assert!(window.push(item("x".to_string(), None)).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() {
|
||||
let mut entry = item("a".to_string(), None);
|
||||
entry.defer(Duration::from_secs(2));
|
||||
let first = entry.first_failure.expect("first failure");
|
||||
tokio::time::advance(Duration::from_secs(29)).await;
|
||||
entry.defer(Duration::from_secs(8));
|
||||
assert_eq!(entry.first_failure, Some(first));
|
||||
assert_eq!(entry.due, first + MAX_DEFERRED_AGE);
|
||||
assert!(!entry.expired());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert!(entry.expired());
|
||||
}
|
||||
@@ -29,7 +29,7 @@ use rustfs_heal::heal::{
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{path::Path, process::Command, sync::Arc, time::Duration};
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
@@ -40,15 +40,10 @@ const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
heal_env_at(None).await
|
||||
}
|
||||
|
||||
async fn heal_env_at(base_dir: Option<&Path>) -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
let mut builder = rustfs_test_utils::TestECStoreEnv::builder().prefix("rustfs_heal_mrf_test");
|
||||
if let Some(base_dir) = base_dir {
|
||||
builder = builder.base_dir(base_dir);
|
||||
}
|
||||
let env = builder.build().await;
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_mrf_test")
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, heal_storage)
|
||||
}
|
||||
@@ -65,29 +60,6 @@ fn make_manager(storage: Arc<dyn HealStorageAPI>) -> Arc<HealManager> {
|
||||
))
|
||||
}
|
||||
|
||||
async fn register_local_disks(disk_paths: &[std::path::PathBuf], cmd_line: &str) {
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: cmd_line.to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
}
|
||||
|
||||
/// Encode one journal record independently of the implementation, so a format
|
||||
/// drift between writer and this fixture fails loudly here.
|
||||
fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]>, attempts: u8) -> Vec<u8> {
|
||||
@@ -122,12 +94,6 @@ fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||
}
|
||||
|
||||
fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool {
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(relative_path).exists())
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
@@ -185,7 +151,26 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
|
||||
// The journal reader resolves disks through the process-local disk map;
|
||||
// register the environment's disks the same way server startup does.
|
||||
register_local_disks(&disk_paths, "mrf-test").await;
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "mrf-test".to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
|
||||
let mut journal = journal_record(1, "replay-bucket", "replay-object", Some([9u8; 16]), 0);
|
||||
journal.extend(journal_record(3, "replay-bucket", "partial-object", None, 1));
|
||||
@@ -228,7 +213,26 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
#[serial]
|
||||
async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-authoritative-test").await;
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "mrf-authoritative-test".to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
|
||||
let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0);
|
||||
let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
|
||||
@@ -260,133 +264,3 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
/// If replay reaches a full heal-manager queue, the old journal remains the
|
||||
/// durable restart anchor until a later consumer flush publishes the pending
|
||||
/// successor snapshot.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_retains_file_when_manager_is_full() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-full-replay-test").await;
|
||||
|
||||
let mut journal = journal_record(1, "full-bucket", "first-object", None, 0);
|
||||
journal.extend(journal_record(1, "full-bucket", "second-object", None, 0));
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &journal);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &journal);
|
||||
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 2, "both records must be decoded before manager admission");
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"only the first record can enter a one-slot manager queue"
|
||||
);
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||
"replay must keep the authoritative journal when a later record is pending retry"
|
||||
);
|
||||
|
||||
let restarted = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
let replayed_after_restart = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(
|
||||
replayed_after_restart, 2,
|
||||
"retained startup journal must replay again after a process restart"
|
||||
);
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the restart sees the same bounded admission state instead of a lost tail"
|
||||
);
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||
"the anchor remains until a successor snapshot can safely replace it"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_journal_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_MRF_REPLAY_CHILD_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let (disk_paths, _storage) = heal_env_at(Some(Path::new(&root))).await;
|
||||
let mut journal = journal_record(1, "child-restart-bucket", "first-object", None, 0);
|
||||
journal.extend(journal_record(1, "child-restart-bucket", "second-object", None, 0));
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &journal);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &journal);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL),
|
||||
"child process must publish the authoritative MRF journal before exiting"
|
||||
);
|
||||
});
|
||||
std::process::exit(77);
|
||||
}
|
||||
|
||||
/// A journal published by a different OS process must remain a durable anchor
|
||||
/// when the restarted process can only admit a prefix of the replayed intents.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_retains_child_process_anchor_when_manager_is_full() {
|
||||
let temp_dir = tempfile::tempdir().expect("child process MRF root");
|
||||
let status = Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("mrf_journal_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_MRF_REPLAY_CHILD_ROOT", temp_dir.path())
|
||||
.status()
|
||||
.expect("child MRF fixture should start");
|
||||
assert_eq!(status.code(), Some(77), "child process did not reach the MRF journal boundary");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL),
|
||||
"restarted process must see the authoritative MRF journal left by the child"
|
||||
);
|
||||
|
||||
let restarted = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 2, "the restarted process must decode the complete child journal");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"bounded admission may accept only the prefix, but must not lose the replayed tail"
|
||||
);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL),
|
||||
"replay must retain the child-published journal until a successor snapshot can replace it"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -86,7 +86,6 @@ rustfs-ecstore = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-policy.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
serde_with = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
thiserror.workspace = true
|
||||
arc-swap = { workspace = true }
|
||||
|
||||
+54
-654
@@ -19,13 +19,11 @@
|
||||
//! and ID token verification.
|
||||
|
||||
use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
|
||||
use openidconnect::core::{
|
||||
CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm,
|
||||
};
|
||||
use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreJsonWebKeySet};
|
||||
use openidconnect::{
|
||||
AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, DiscoveryError, IssuerUrl,
|
||||
JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl,
|
||||
ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope, TokenUrl,
|
||||
ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope,
|
||||
};
|
||||
use reqwest::{Certificate, Client};
|
||||
use rustfs_config::oidc::*;
|
||||
@@ -54,7 +52,6 @@ const EVENT_OIDC_HTTP: &str = "oidc_http";
|
||||
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
|
||||
const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
|
||||
const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
|
||||
const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy";
|
||||
const OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY: &str = "OIDC provider discovery blocked by outbound policy";
|
||||
const OIDC_HTTP_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(10);
|
||||
const OIDC_HTTP_CONNECT_TIMEOUT: StdDuration = StdDuration::from_secs(3);
|
||||
@@ -756,7 +753,7 @@ pub struct SourcedOidcProviderConfig {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct OidcProviderValidationResult {
|
||||
pub issuer: String,
|
||||
pub authorization_endpoint: Option<String>,
|
||||
pub authorization_endpoint: String,
|
||||
pub token_endpoint: Option<String>,
|
||||
}
|
||||
|
||||
@@ -786,142 +783,10 @@ pub struct OidcClaims {
|
||||
/// on-the-fly from metadata when needed.
|
||||
#[derive(Clone)]
|
||||
struct ProviderState {
|
||||
metadata: DiscoveredProviderMetadata,
|
||||
metadata: ProviderMetadataWithLogout,
|
||||
discovered_at: Instant,
|
||||
}
|
||||
|
||||
// Workload issuers do not implement the browser authorization flow. Keep their
|
||||
// verification metadata separate rather than inventing an authorization URL.
|
||||
#[serde_with::serde_as]
|
||||
#[derive(Clone, Deserialize)]
|
||||
struct WorkloadProviderMetadata {
|
||||
issuer: IssuerUrl,
|
||||
jwks_uri: JsonWebKeySetUrl,
|
||||
token_endpoint: Option<TokenUrl>,
|
||||
#[serde_as(as = "serde_with::VecSkipError<_>")]
|
||||
id_token_signing_alg_values_supported: Vec<CoreJwsSigningAlgorithm>,
|
||||
#[serde(skip)]
|
||||
jwks: CoreJsonWebKeySet,
|
||||
// Discovery is extensible; report unsupported fields without logging values.
|
||||
#[serde(flatten)]
|
||||
additional_fields: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum DiscoveredProviderMetadata {
|
||||
Console(Box<ProviderMetadataWithLogout>),
|
||||
Workload(Box<WorkloadProviderMetadata>),
|
||||
}
|
||||
|
||||
impl DiscoveredProviderMetadata {
|
||||
fn parse(body: &[u8], hide_from_ui: bool) -> Result<Self, String> {
|
||||
let document: serde_json::Value = serde_json::from_slice(body).map_err(|err| err.to_string())?;
|
||||
if hide_from_ui
|
||||
&& document
|
||||
.as_object()
|
||||
.is_some_and(|fields| !fields.contains_key("authorization_endpoint"))
|
||||
{
|
||||
let mut metadata: WorkloadProviderMetadata = serde_json::from_slice(body).map_err(|err| err.to_string())?;
|
||||
if !metadata.additional_fields.is_empty() {
|
||||
warn!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
result = "workload_discovery_additional_fields",
|
||||
field_count = metadata.additional_fields.len(),
|
||||
"workload discovery contains additional fields"
|
||||
);
|
||||
metadata.additional_fields.clear();
|
||||
}
|
||||
Ok(Self::Workload(Box::new(metadata)))
|
||||
} else {
|
||||
serde_json::from_slice(body)
|
||||
.map(|metadata| Self::Console(Box::new(metadata)))
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn console(&self) -> Result<&ProviderMetadataWithLogout, String> {
|
||||
match self {
|
||||
Self::Console(metadata) => Ok(metadata),
|
||||
Self::Workload(_) => Err("OIDC provider has no authorization endpoint; only web identity is supported".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn issuer(&self) -> &IssuerUrl {
|
||||
match self {
|
||||
Self::Console(metadata) => metadata.issuer(),
|
||||
Self::Workload(metadata) => &metadata.issuer,
|
||||
}
|
||||
}
|
||||
|
||||
fn jwks_uri(&self) -> &JsonWebKeySetUrl {
|
||||
match self {
|
||||
Self::Console(metadata) => metadata.jwks_uri(),
|
||||
Self::Workload(metadata) => &metadata.jwks_uri,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_jwks(self, jwks: CoreJsonWebKeySet) -> Self {
|
||||
match self {
|
||||
Self::Console(metadata) => Self::Console(Box::new(metadata.set_jwks(jwks))),
|
||||
Self::Workload(mut metadata) => {
|
||||
metadata.jwks = jwks;
|
||||
Self::Workload(metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn authorization_endpoint(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Console(metadata) => Some(metadata.authorization_endpoint().to_string()),
|
||||
Self::Workload(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn token_endpoint(&self) -> Option<&TokenUrl> {
|
||||
match self {
|
||||
Self::Console(metadata) => metadata.token_endpoint(),
|
||||
Self::Workload(metadata) => metadata.token_endpoint.as_ref(),
|
||||
}
|
||||
}
|
||||
|
||||
fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> {
|
||||
let client_id = ClientId::new(config.client_id.clone());
|
||||
let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone()));
|
||||
let (issuer, jwks, algorithms) = match self {
|
||||
Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()),
|
||||
Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported),
|
||||
};
|
||||
let verifier = match secret {
|
||||
Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()),
|
||||
None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()),
|
||||
};
|
||||
verifier.set_allowed_algs(algorithms.clone())
|
||||
}
|
||||
}
|
||||
|
||||
// This adapter is used only for discovery/JWKS fetches, never token exchange.
|
||||
struct JwksAcceptClient<'a> {
|
||||
inner: &'a ReqwestHttpClient,
|
||||
discovery_url: Option<Url>,
|
||||
}
|
||||
|
||||
impl<'c> AsyncHttpClient<'c> for JwksAcceptClient<'_> {
|
||||
type Error = OidcHttpError;
|
||||
type Future = <ReqwestHttpClient as AsyncHttpClient<'c>>::Future;
|
||||
|
||||
fn call(&'c self, mut request: http::Request<Vec<u8>>) -> Self::Future {
|
||||
if !self.discovery_url.as_ref().is_some_and(|url| request.uri() == url.as_str()) {
|
||||
request.headers_mut().insert(
|
||||
http::header::ACCEPT,
|
||||
http::HeaderValue::from_static("application/json, application/jwk-set+json"),
|
||||
);
|
||||
}
|
||||
self.inner.call(request)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderState {
|
||||
fn is_stale(&self) -> bool {
|
||||
self.discovered_at.elapsed() >= OIDC_JWKS_REFRESH_INTERVAL
|
||||
@@ -1067,7 +932,7 @@ impl OidcSys {
|
||||
let redirect = RedirectUrl::new(redirect_uri.to_string()).map_err(|e| format!("invalid redirect URI: {e}"))?;
|
||||
|
||||
let client = CoreClient::from_provider_metadata(
|
||||
state.metadata.console()?.clone(),
|
||||
state.metadata.clone(),
|
||||
ClientId::new(config.client_id.clone()),
|
||||
config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
|
||||
)
|
||||
@@ -1129,7 +994,7 @@ impl OidcSys {
|
||||
|
||||
// Construct CoreClient on-the-fly with JWKS from discovery
|
||||
let client = CoreClient::from_provider_metadata(
|
||||
provider_state.metadata.console()?.clone(),
|
||||
provider_state.metadata.clone(),
|
||||
ClientId::new(config.client_id.clone()),
|
||||
config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
|
||||
)
|
||||
@@ -1365,7 +1230,7 @@ impl OidcSys {
|
||||
);
|
||||
|
||||
let client = CoreClient::from_provider_metadata(
|
||||
refreshed_state.metadata.console()?.clone(),
|
||||
refreshed_state.metadata,
|
||||
ClientId::new(config.client_id.clone()),
|
||||
config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
|
||||
)
|
||||
@@ -1455,7 +1320,7 @@ impl OidcSys {
|
||||
.get(&session.provider_id)
|
||||
.ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?;
|
||||
let state = self.ensure_provider_state(&session.provider_id, config).await?;
|
||||
let Some(end_session_endpoint) = state.metadata.console()?.additional_metadata().end_session_endpoint.clone() else {
|
||||
let Some(end_session_endpoint) = state.metadata.additional_metadata().end_session_endpoint.clone() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
@@ -1595,6 +1460,14 @@ impl OidcSys {
|
||||
|
||||
state = self.ensure_provider_state_if_stale(&provider_id, &config, &state).await?;
|
||||
|
||||
// Reconstruct CoreClient from provider metadata
|
||||
let client = CoreClient::from_provider_metadata(
|
||||
state.metadata.clone(),
|
||||
ClientId::new(config.client_id.clone()),
|
||||
config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
|
||||
)
|
||||
.set_auth_type(AuthType::RequestBody);
|
||||
|
||||
// Parse raw JWT string into CoreIdToken
|
||||
let id_token: CoreIdToken = jwt
|
||||
.parse()
|
||||
@@ -1602,9 +1475,8 @@ impl OidcSys {
|
||||
|
||||
// Verify the token (signature, issuer, audience, expiry) — skip nonce
|
||||
// (nonce is only required for the authorization code flow)
|
||||
let verifier = state
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
state = self
|
||||
@@ -1614,9 +1486,14 @@ impl OidcSys {
|
||||
format!("ID token verification failed: {e}; failed to refresh provider metadata: {refresh_err}")
|
||||
})?;
|
||||
|
||||
let verifier = state
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
let client = CoreClient::from_provider_metadata(
|
||||
state.metadata,
|
||||
ClientId::new(config.client_id.clone()),
|
||||
config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
|
||||
)
|
||||
.set_auth_type(AuthType::RequestBody);
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
id_token
|
||||
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
|
||||
@@ -1991,39 +1868,18 @@ impl OidcSys {
|
||||
let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
|
||||
|
||||
for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
|
||||
let discovered = if config.hide_from_ui {
|
||||
Self::discover_provider_from_config_url(config, candidate_issuer, http_client).await
|
||||
} else {
|
||||
let client = JwksAcceptClient {
|
||||
inner: http_client,
|
||||
discovery_url: Some(
|
||||
issuer_url
|
||||
.join(".well-known/openid-configuration")
|
||||
.map_err(|err| err.to_string())?,
|
||||
),
|
||||
};
|
||||
ProviderMetadataWithLogout::discover_async(issuer_url.clone(), &client)
|
||||
.await
|
||||
.map(|metadata| ProviderState {
|
||||
metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)),
|
||||
match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client).await {
|
||||
Ok(metadata) => {
|
||||
return Ok(ProviderState {
|
||||
metadata,
|
||||
discovered_at: Instant::now(),
|
||||
})
|
||||
.map_err(|err| match err {
|
||||
DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason)) => {
|
||||
format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}")
|
||||
}
|
||||
err => format!("discovery failed: {err}"),
|
||||
})
|
||||
};
|
||||
match discovered {
|
||||
Ok(state) => return Ok(state),
|
||||
Err(error)
|
||||
if error.starts_with(OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY)
|
||||
|| error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY) =>
|
||||
{
|
||||
return Err(error);
|
||||
});
|
||||
}
|
||||
Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => {
|
||||
return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}"));
|
||||
}
|
||||
Err(error) => {
|
||||
let error = format!("discovery failed: {error}");
|
||||
let is_transient_transport = error.contains("Request failed");
|
||||
let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES;
|
||||
if should_retry {
|
||||
@@ -2077,14 +1933,7 @@ impl OidcSys {
|
||||
http_client: &ReqwestHttpClient,
|
||||
) -> Result<ProviderState, String> {
|
||||
let issuer_url = IssuerUrl::new(issuer.trim().to_string()).map_err(|e| format!("invalid issuer URL: {e}"))?;
|
||||
let explicit_issuer = config.issuer.as_deref().is_some_and(|issuer| !issuer.trim().is_empty());
|
||||
let discovery_url = if explicit_issuer {
|
||||
discovery_url_from_config_url(&config.config_url)?
|
||||
} else {
|
||||
issuer_url
|
||||
.join(".well-known/openid-configuration")
|
||||
.map_err(|err| err.to_string())?
|
||||
};
|
||||
let discovery_url = discovery_url_from_config_url(&config.config_url)?;
|
||||
let request = http::Request::builder()
|
||||
.uri(discovery_url.to_string())
|
||||
.method(http::Method::GET)
|
||||
@@ -2097,25 +1946,13 @@ impl OidcSys {
|
||||
Err(OidcHttpError::ForbiddenOutbound(reason)) => {
|
||||
return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}"));
|
||||
}
|
||||
Err(err) => return Err(format!("discovery request failed: Request failed: {err}")),
|
||||
Err(err) => return Err(format!("discovery request failed: {err}")),
|
||||
};
|
||||
if response.status() != http::StatusCode::OK {
|
||||
return Err(format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url));
|
||||
}
|
||||
|
||||
if !explicit_issuer
|
||||
&& let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE)
|
||||
&& !content_type.to_str().ok().is_some_and(|value| {
|
||||
value
|
||||
.split(';')
|
||||
.next()
|
||||
.is_some_and(|essence| essence.eq_ignore_ascii_case("application/json"))
|
||||
})
|
||||
{
|
||||
return Err("Unexpected response Content-Type: expected application/json".into());
|
||||
}
|
||||
|
||||
let provider_metadata = DiscoveredProviderMetadata::parse(response.body(), config.hide_from_ui)
|
||||
let provider_metadata = serde_json::from_slice::<ProviderMetadataWithLogout>(response.body())
|
||||
.map_err(|err| format!("failed to parse discovery response: {err}"))?;
|
||||
if provider_metadata.issuer() != &issuer_url {
|
||||
return Err(format!(
|
||||
@@ -2125,23 +1962,11 @@ impl OidcSys {
|
||||
));
|
||||
}
|
||||
|
||||
let jwks_url = if explicit_issuer {
|
||||
jwks_url_from_config_url(&config.config_url, &issuer_url, provider_metadata.jwks_uri())?
|
||||
} else {
|
||||
provider_metadata.jwks_uri().clone()
|
||||
};
|
||||
let jwks = match CoreJsonWebKeySet::fetch_async(
|
||||
&jwks_url,
|
||||
&JwksAcceptClient {
|
||||
inner: http_client,
|
||||
discovery_url: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let jwks_url = jwks_url_from_config_url(&config.config_url, &issuer_url, provider_metadata.jwks_uri())?;
|
||||
let jwks = match CoreJsonWebKeySet::fetch_async(&jwks_url, http_client).await {
|
||||
Ok(jwks) => jwks,
|
||||
Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => {
|
||||
return Err(format!("{OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY}: {reason}"));
|
||||
return Err(format!("JWKS request blocked by outbound policy: {reason}"));
|
||||
}
|
||||
Err(err) => return Err(format!("failed to fetch JWKS: {err}")),
|
||||
};
|
||||
@@ -2213,7 +2038,7 @@ pub async fn validate_oidc_provider_config_with_extra_root_ca(
|
||||
|
||||
Ok(OidcProviderValidationResult {
|
||||
issuer: state.metadata.issuer().to_string(),
|
||||
authorization_endpoint: state.metadata.authorization_endpoint(),
|
||||
authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
|
||||
token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
|
||||
})
|
||||
}
|
||||
@@ -2773,7 +2598,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_mock_oidc_request(stream: &mut impl std::io::Read) -> String {
|
||||
fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String {
|
||||
let mut request_bytes = Vec::new();
|
||||
let mut buffer = [0u8; 4096];
|
||||
loop {
|
||||
@@ -2790,11 +2615,8 @@ mod tests {
|
||||
break;
|
||||
}
|
||||
}
|
||||
String::from_utf8_lossy(&request_bytes).into_owned()
|
||||
}
|
||||
|
||||
fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String {
|
||||
read_mock_oidc_request(stream)
|
||||
let request = String::from_utf8_lossy(&request_bytes);
|
||||
request
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
@@ -2805,7 +2627,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String {
|
||||
let (status, body) = if path.ends_with("/.well-known/openid-configuration") {
|
||||
let (status, body) = if path.contains("/.well-known/openid-configuration") {
|
||||
(200, discovery_body)
|
||||
} else if path == expected_jwks_path {
|
||||
(200, jwks_body)
|
||||
@@ -2824,7 +2646,6 @@ mod tests {
|
||||
build_discovery_issuer: F,
|
||||
max_requests: usize,
|
||||
signing_alg: &'static str,
|
||||
workload: bool,
|
||||
jwks_response: J,
|
||||
) -> Option<(String, std::thread::JoinHandle<()>)>
|
||||
where
|
||||
@@ -2849,7 +2670,7 @@ mod tests {
|
||||
};
|
||||
let base = format!("http://{}", listener.local_addr().expect("listener local address should be available"));
|
||||
let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base);
|
||||
let mut discovery_document = serde_json::json!({
|
||||
let discovery_body = serde_json::json!({
|
||||
"issuer": discovery_issuer,
|
||||
"authorization_endpoint": format!("{base}/authorize"),
|
||||
"token_endpoint": format!("{base}/token"),
|
||||
@@ -2858,14 +2679,8 @@ mod tests {
|
||||
"response_modes_supported": ["query"],
|
||||
"subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": [signing_alg],
|
||||
});
|
||||
if workload {
|
||||
let fields = discovery_document.as_object_mut().expect("mock metadata is an object");
|
||||
fields.remove("authorization_endpoint");
|
||||
fields.remove("token_endpoint");
|
||||
fields.insert("response_types_supported".into(), serde_json::json!(["id_token"]));
|
||||
}
|
||||
let discovery_body = discovery_document.to_string();
|
||||
})
|
||||
.to_string();
|
||||
let (ready_tx, ready_rx) = mpsc::channel();
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
@@ -2907,41 +2722,12 @@ mod tests {
|
||||
.set_read_timeout(Some(Duration::from_secs(1)))
|
||||
.expect("failed to set discovery mock read timeout");
|
||||
|
||||
let request = read_mock_oidc_request(&mut stream);
|
||||
let path = request
|
||||
.lines()
|
||||
.next()
|
||||
.and_then(|line| line.split_whitespace().nth(1))
|
||||
.unwrap_or("");
|
||||
let path = read_mock_oidc_request_path(&mut stream);
|
||||
let jwks_body = jwks_response(jwks_fetches);
|
||||
if path == expected_jwks_path {
|
||||
jwks_fetches += 1;
|
||||
}
|
||||
let mut response = mock_oidc_response(path, &discovery_body, &expected_jwks_path, &jwks_body);
|
||||
if path.contains("/.well-known/openid-configuration") {
|
||||
assert!(
|
||||
request
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.any(|(name, value)| { name.eq_ignore_ascii_case("accept") && value.trim() == "application/json" }),
|
||||
"discovery Accept must remain unchanged"
|
||||
);
|
||||
}
|
||||
if path == expected_jwks_path {
|
||||
let expected_type = if workload {
|
||||
"application/jwk-set+json"
|
||||
} else {
|
||||
"application/json"
|
||||
};
|
||||
let accepts_type = request.lines().filter_map(|line| line.split_once(':')).any(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("accept") && value.split(',').any(|item| item.trim() == expected_type)
|
||||
});
|
||||
if !accepts_type {
|
||||
response = "HTTP/1.1 406 Not Acceptable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into();
|
||||
} else if workload {
|
||||
response = response.replace("Content-Type: application/json", "Content-Type: application/jwk-set+json");
|
||||
}
|
||||
}
|
||||
let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, &jwks_body);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = stream.shutdown(Shutdown::Both);
|
||||
@@ -2966,7 +2752,7 @@ mod tests {
|
||||
where
|
||||
F: Fn(&str) -> (String, String, String) + Send + 'static,
|
||||
{
|
||||
start_mock_oidc_discovery_server_with_jwks(build_discovery_issuer, max_requests, "RS256", false, |_| {
|
||||
start_mock_oidc_discovery_server_with_jwks(build_discovery_issuer, max_requests, "RS256", |_| {
|
||||
r#"{"keys":[]}"#.to_string()
|
||||
})
|
||||
}
|
||||
@@ -2993,7 +2779,6 @@ mod tests {
|
||||
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
4,
|
||||
"ES256",
|
||||
false,
|
||||
move |fetch| {
|
||||
if fetch == 0 {
|
||||
initial_jwks.clone()
|
||||
@@ -3048,391 +2833,6 @@ mod tests {
|
||||
handle.join().expect("rotating JWKS mock server should exit cleanly");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_provider_console_login_preserves_hidden_and_issuer_modes() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
for hidden in [false, true] {
|
||||
for explicit in [false, true] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let issuer = if explicit {
|
||||
"https://issuer.example.com".to_string()
|
||||
} else {
|
||||
base.clone()
|
||||
};
|
||||
let mut config =
|
||||
build_mocked_oidc_provider_config("console", &format!("{base}/.well-known/openid-configuration"));
|
||||
config.hide_from_ui = hidden;
|
||||
config.issuer = explicit.then(|| issuer.clone());
|
||||
config.client_secret = Some(Nonce::new_random().secret().clone());
|
||||
let server_config = config.clone();
|
||||
let server_base = base.clone();
|
||||
let redirect = "https://console.example.com/oauth_callback";
|
||||
let (key, jwk) = oidc_es256_key_and_jwk("console");
|
||||
let (auth_tx, auth_rx) = tokio::sync::oneshot::channel::<HashMap<String, String>>();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut auth_rx = Some(auth_rx);
|
||||
for expected_path in ["/.well-known/openid-configuration", "/jwks", "/token"] {
|
||||
let (mut stream, _) = tokio::time::timeout(StdDuration::from_secs(10), listener.accept())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
let header_end = loop {
|
||||
bytes.push(stream.read_u8().await.unwrap());
|
||||
assert!(bytes.len() < 8192);
|
||||
if bytes.ends_with(b"\r\n\r\n") {
|
||||
break bytes.len();
|
||||
}
|
||||
};
|
||||
let headers = String::from_utf8(bytes).unwrap();
|
||||
let request_line = headers.lines().next().unwrap();
|
||||
assert_eq!(request_line.split_whitespace().nth(1), Some(expected_path));
|
||||
let headers_map: HashMap<_, _> = headers
|
||||
.lines()
|
||||
.skip(1)
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string()))
|
||||
.collect();
|
||||
let body = match expected_path {
|
||||
"/.well-known/openid-configuration" => {
|
||||
assert!(request_line.starts_with("GET "));
|
||||
assert_eq!(headers_map["accept"], "application/json");
|
||||
serde_json::json!({
|
||||
"issuer": issuer, "authorization_endpoint": format!("{server_base}/authorize"),
|
||||
"token_endpoint": format!("{server_base}/token"), "jwks_uri": format!("{server_base}/jwks"),
|
||||
"response_types_supported": ["code"], "subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": ["ES256"]
|
||||
})
|
||||
}
|
||||
"/jwks" => {
|
||||
assert!(request_line.starts_with("GET "));
|
||||
assert!(headers_map["accept"].contains("application/json"));
|
||||
serde_json::json!({"keys": [jwk]})
|
||||
}
|
||||
"/token" => {
|
||||
assert!(request_line.starts_with("POST "));
|
||||
assert_eq!(headers_map["accept"], "application/json");
|
||||
assert!(headers_map["content-type"].starts_with("application/x-www-form-urlencoded"));
|
||||
let length: usize = headers_map["content-length"].parse().unwrap();
|
||||
assert!(header_end + length < 16384);
|
||||
let mut body = vec![0; length];
|
||||
stream.read_exact(&mut body).await.unwrap();
|
||||
let form: HashMap<String, String> = url::form_urlencoded::parse(&body).into_owned().collect();
|
||||
assert_eq!(form["grant_type"], "authorization_code");
|
||||
assert_eq!(form["code"], "test-authorization-code");
|
||||
assert_eq!(form["client_id"], server_config.client_id);
|
||||
assert_eq!(Some(&form["client_secret"]), server_config.client_secret.as_ref());
|
||||
assert_eq!(form["redirect_uri"], redirect);
|
||||
let auth = auth_rx.take().unwrap().await.unwrap();
|
||||
let challenge = PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new(form["code_verifier"].clone()));
|
||||
assert_eq!(challenge.as_str(), auth["code_challenge"]);
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let mut header = Header::new(Algorithm::ES256);
|
||||
header.kid = Some("console".into());
|
||||
let token = jsonwebtoken::encode(&header, &serde_json::json!({
|
||||
"iss": issuer, "sub": "existing-user", "aud": server_config.client_id,
|
||||
"iat": now, "exp": now + 300, "nonce": auth["nonce"],
|
||||
"email": "user@example.com", "groups": ["readwrite"]
|
||||
}), &key).unwrap();
|
||||
serde_json::json!({"access_token": "test-access-token", "token_type": "Bearer", "id_token": token})
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}.to_string();
|
||||
stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap();
|
||||
}
|
||||
});
|
||||
let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap());
|
||||
let discovered = OidcSys::discover_provider(&config, &http_client).await.unwrap();
|
||||
let sys = OidcSys {
|
||||
configs: HashMap::from([(config.id.clone(), config)]),
|
||||
provider_states: RwLock::new(HashMap::from([("console".into(), discovered)])),
|
||||
state_store: OidcStateStore::new(),
|
||||
http_client,
|
||||
};
|
||||
let auth_url = sys.authorize_url("console", redirect, Some("/buckets".into())).await.unwrap();
|
||||
let auth_url = Url::parse(&auth_url).unwrap();
|
||||
assert_eq!(auth_url.as_str().split('?').next(), Some(format!("{base}/authorize").as_str()));
|
||||
let auth: HashMap<String, String> = auth_url.query_pairs().into_owned().collect();
|
||||
assert_eq!(auth["response_type"], "code");
|
||||
assert_eq!(auth["client_id"], "rustfs-oidc-test");
|
||||
assert!(!auth["nonce"].is_empty());
|
||||
assert!(!auth["state"].is_empty());
|
||||
assert_eq!(auth["redirect_uri"], redirect);
|
||||
assert_eq!(auth["code_challenge_method"], "S256");
|
||||
assert!(auth["scope"].split_whitespace().any(|scope| scope == "openid"));
|
||||
let state = auth["state"].clone();
|
||||
auth_tx.send(auth).unwrap();
|
||||
let (claims, provider, session, _) = sys
|
||||
.exchange_code(&state, "test-authorization-code", redirect)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}"));
|
||||
assert_eq!(provider, "console");
|
||||
assert_eq!(claims.sub, "existing-user");
|
||||
assert_eq!(claims.email, "user@example.com");
|
||||
assert_eq!(claims.groups, vec!["readwrite"]);
|
||||
assert_eq!(session.redirect_after.as_deref(), Some("/buckets"));
|
||||
assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await,
|
||||
Err(error) if error == "invalid or expired OIDC state"));
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workload_metadata_requires_hidden_provider_and_valid_verification_fields() {
|
||||
let document = serde_json::json!({
|
||||
"issuer": "https://issuer.example.com",
|
||||
"jwks_uri": "https://issuer.example.com/jwks",
|
||||
"response_types_supported": ["id_token"],
|
||||
"subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": ["ES256"],
|
||||
});
|
||||
let parse =
|
||||
|value: &serde_json::Value, hidden| DiscoveredProviderMetadata::parse(&serde_json::to_vec(value).unwrap(), hidden);
|
||||
let metadata = parse(&document, true).expect("hidden workload metadata should parse");
|
||||
assert!(metadata.authorization_endpoint().is_none());
|
||||
assert!(metadata.console().err().unwrap().contains("only web identity"));
|
||||
assert!(parse(&document, false).err().unwrap().contains("authorization_endpoint"));
|
||||
for field in ["issuer", "jwks_uri", "id_token_signing_alg_values_supported"] {
|
||||
let mut invalid = document.clone();
|
||||
invalid.as_object_mut().unwrap().remove(field);
|
||||
assert!(parse(&invalid, true).err().unwrap().contains(field), "missing {field}");
|
||||
}
|
||||
for endpoint in [serde_json::Value::Null, serde_json::json!(""), serde_json::json!("not a URL")] {
|
||||
let mut invalid = document.clone();
|
||||
invalid["authorization_endpoint"] = endpoint;
|
||||
assert!(parse(&invalid, true).is_err(), "invalid endpoint must not select workload metadata");
|
||||
}
|
||||
let mut complete = document;
|
||||
complete["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize");
|
||||
for hidden in [true, false] {
|
||||
let metadata = parse(&complete, hidden).expect("full providers keep the existing parser");
|
||||
assert!(metadata.console().is_ok());
|
||||
assert!(metadata.token_endpoint().is_none(), "token endpoint remains optional");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workload_metadata_rejects_duplicate_fields() {
|
||||
for hidden in [false, true] {
|
||||
let document = format!(
|
||||
r#"{{"issuer":"https://wrong.example.com","issuer":"https://issuer.example.com",{}"jwks_uri":"https://issuer.example.com/jwks","response_types_supported":["id_token"],"subject_types_supported":["public"],"id_token_signing_alg_values_supported":["ES256"]}}"#,
|
||||
if hidden {
|
||||
""
|
||||
} else {
|
||||
r#""authorization_endpoint":"https://issuer.example.com/authorize","#
|
||||
},
|
||||
);
|
||||
let error = DiscoveredProviderMetadata::parse(document.as_bytes(), hidden)
|
||||
.err()
|
||||
.expect("duplicate issuer must fail");
|
||||
assert!(error.contains("duplicate field"), "{error}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
|
||||
let secret = Nonce::new_random().secret().to_string();
|
||||
let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com");
|
||||
config.client_secret = Some(secret.clone());
|
||||
config.other_audiences = vec!["additional-audience".into()];
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({
|
||||
"iss": config.config_url, "sub": "repo:example/project:ref:refs/heads/main",
|
||||
"aud": [config.client_id, "additional-audience"], "iat": now, "exp": now + 300,
|
||||
});
|
||||
let signed =
|
||||
jsonwebtoken::encode(&Header::new(Algorithm::HS256), &payload, &EncodingKey::from_secret(secret.as_bytes())).unwrap();
|
||||
let token: CoreIdToken = signed.parse().unwrap();
|
||||
for workload in [false, true] {
|
||||
for (algorithms, accepted) in [
|
||||
(serde_json::json!(["HS256", "unsupported-future-algorithm"]), true),
|
||||
(serde_json::json!(["ES256"]), false),
|
||||
(serde_json::json!(["unsupported-future-algorithm"]), false),
|
||||
] {
|
||||
let mut document = serde_json::json!({
|
||||
"issuer": config.config_url, "jwks_uri": "https://issuer.example.com/jwks",
|
||||
"response_types_supported": ["id_token"], "subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": algorithms,
|
||||
});
|
||||
if !workload {
|
||||
document["authorization_endpoint"] = serde_json::json!("https://issuer.example.com/authorize");
|
||||
}
|
||||
let metadata = DiscoveredProviderMetadata::parse(&serde_json::to_vec(&document).unwrap(), true).unwrap();
|
||||
let verifier = metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
assert_eq!(
|
||||
token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
accepted,
|
||||
"workload={workload}, algorithms={algorithms}"
|
||||
);
|
||||
if accepted {
|
||||
assert!(
|
||||
token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"additional audiences require explicit trust"
|
||||
);
|
||||
let mut wrong_secret = config.clone();
|
||||
wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string());
|
||||
let verifier = metadata
|
||||
.verifier(&wrong_secret)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
assert!(
|
||||
token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"incorrect client secret must fail"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workload_discovery_stops_after_forbidden_jwks() {
|
||||
let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let (base, handle) = start_mock_oidc_discovery_server_with_jwks(
|
||||
|base| (base.to_string(), "http://192.168.65.254:8080/jwks".into(), "/jwks".into()),
|
||||
2,
|
||||
"ES256",
|
||||
true,
|
||||
move |_| {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
serde_json::json!({"keys": []}).to_string()
|
||||
},
|
||||
)
|
||||
.expect("workload discovery mock must bind");
|
||||
let mut config = build_mocked_oidc_provider_config("workload", &base);
|
||||
config.hide_from_ui = true;
|
||||
let client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap());
|
||||
let error = OidcSys::discover_provider(&config, &client)
|
||||
.await
|
||||
.err()
|
||||
.expect("private JWKS must be blocked");
|
||||
assert!(error.starts_with(OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY), "{error}");
|
||||
handle.join().unwrap();
|
||||
assert_eq!(
|
||||
requests.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"a policy denial must not retry discovery"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workload_config_validation_reports_absent_console_endpoints() {
|
||||
let (base, handle) = start_mock_oidc_discovery_server_with_jwks(
|
||||
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()),
|
||||
2,
|
||||
"ES256",
|
||||
true,
|
||||
|_| serde_json::json!({"keys": []}).to_string(),
|
||||
)
|
||||
.expect("workload discovery mock must bind");
|
||||
let mut config = build_mocked_oidc_provider_config("workload", &base);
|
||||
config.hide_from_ui = true;
|
||||
// Inferred issuers keep the library's discovery URL construction.
|
||||
config.config_url = format!("{base}/.well-known/openid-configuration?ignored=1#ignored");
|
||||
let result = validate_mocked_oidc_provider_config(&config)
|
||||
.await
|
||||
.expect("hidden workload configuration should validate");
|
||||
assert_eq!(result.issuer, base);
|
||||
assert!(result.authorization_endpoint.is_none());
|
||||
assert!(result.token_endpoint.is_none());
|
||||
handle.join().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workload_discovery_verification_and_rotation() {
|
||||
for explicit_issuer in [false, true] {
|
||||
let (_, initial_jwk) = oidc_es256_key_and_jwk("initial");
|
||||
let (key, rotated_jwk) = oidc_es256_key_and_jwk("rotated");
|
||||
let initial_jwks = serde_json::json!({"keys": [initial_jwk]}).to_string();
|
||||
let rotated_jwks = serde_json::json!({"keys": [rotated_jwk]}).to_string();
|
||||
let (base, handle) = start_mock_oidc_discovery_server_with_jwks(
|
||||
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".into()),
|
||||
4,
|
||||
"ES256",
|
||||
true,
|
||||
move |fetch| {
|
||||
if fetch == 0 {
|
||||
initial_jwks.clone()
|
||||
} else {
|
||||
rotated_jwks.clone()
|
||||
}
|
||||
},
|
||||
)
|
||||
.expect("workload discovery mock must bind");
|
||||
let mut config = build_mocked_oidc_provider_config("workload", &base);
|
||||
config.hide_from_ui = true;
|
||||
if explicit_issuer {
|
||||
config.issuer = Some(base.clone());
|
||||
}
|
||||
let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap());
|
||||
let state = OidcSys::discover_provider(&config, &http_client)
|
||||
.await
|
||||
.expect("workload discovery must succeed");
|
||||
assert!(state.metadata.authorization_endpoint().is_none());
|
||||
let sys = OidcSys {
|
||||
configs: HashMap::from([(config.id.clone(), config.clone())]),
|
||||
provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])),
|
||||
state_store: OidcStateStore::new(),
|
||||
http_client,
|
||||
};
|
||||
assert!(sys.has_providers());
|
||||
assert!(sys.list_visible_providers().is_empty());
|
||||
let error = sys
|
||||
.authorize_url(&config.id, "https://console.example.com/callback", None)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("only web identity"));
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({
|
||||
"iss": base, "sub": "system:serviceaccount:default:reader", "aud": [config.client_id],
|
||||
"iat": now, "exp": now + 300, "groups": ["readonly"],
|
||||
"kubernetes.io": {"namespace": "default", "serviceaccount": {"name": "reader"}},
|
||||
});
|
||||
let mut header = Header::new(Algorithm::ES256);
|
||||
header.kid = Some("rotated".into());
|
||||
let token = jsonwebtoken::encode(&header, &payload, &key).unwrap();
|
||||
let (claims, provider) = sys
|
||||
.verify_web_identity_token(&token)
|
||||
.await
|
||||
.expect("rotation must retain workload discovery support");
|
||||
assert_eq!(provider, config.id);
|
||||
assert_eq!(claims.sub, "system:serviceaccount:default:reader");
|
||||
assert_eq!(claims.groups, ["readonly"]);
|
||||
// Repeat after the mock exits: the verified snapshot must be cached.
|
||||
handle.join().unwrap();
|
||||
assert!(sys.verify_web_identity_token(&token).await.is_ok());
|
||||
for (field, value, expected) in [
|
||||
("iss", serde_json::json!("https://wrong.example.com"), "issuer"),
|
||||
("aud", serde_json::json!("wrong-audience"), "audience"),
|
||||
("exp", serde_json::json!(now - 60), "expired"),
|
||||
] {
|
||||
let mut invalid = payload.clone();
|
||||
invalid[field] = value;
|
||||
let token = jsonwebtoken::encode(&header, &invalid, &key).unwrap();
|
||||
let error = sys
|
||||
.verify_web_identity_token(&token)
|
||||
.await
|
||||
.expect_err("invalid workload token must fail");
|
||||
assert!(error.to_lowercase().contains(expected), "{field}: {error}");
|
||||
}
|
||||
let (wrong_key, _) = oidc_es256_key_and_jwk("wrong");
|
||||
let invalid = jsonwebtoken::encode(&header, &payload, &wrong_key).unwrap();
|
||||
let error = sys
|
||||
.verify_web_identity_token(&invalid)
|
||||
.await
|
||||
.expect_err("wrong signature must fail");
|
||||
assert!(error.to_lowercase().contains("signature"), "{error}");
|
||||
assert!(
|
||||
sys.verify_web_identity_token(&token).await.is_ok(),
|
||||
"failed refresh must preserve the cached keys"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn start_mock_oidc_tls_discovery_server<F>(
|
||||
build_discovery_issuer: F,
|
||||
max_requests: usize,
|
||||
@@ -3558,7 +2958,7 @@ mod tests {
|
||||
|
||||
Ok(OidcProviderValidationResult {
|
||||
issuer: state.metadata.issuer().to_string(),
|
||||
authorization_endpoint: state.metadata.authorization_endpoint(),
|
||||
authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
|
||||
token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
|
||||
})
|
||||
}
|
||||
@@ -4158,7 +3558,7 @@ mod tests {
|
||||
provider_states: RwLock::new(HashMap::from([(
|
||||
provider_id.to_string(),
|
||||
ProviderState {
|
||||
metadata: DiscoveredProviderMetadata::Console(Box::new(metadata)),
|
||||
metadata,
|
||||
discovered_at: Instant::now(),
|
||||
},
|
||||
)])),
|
||||
|
||||
@@ -66,8 +66,9 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
|
||||
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
|
||||
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
|
||||
const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer";
|
||||
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str =
|
||||
"Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And";
|
||||
const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates";
|
||||
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str = "Filter has too many predicates";
|
||||
const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key";
|
||||
const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters";
|
||||
const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative";
|
||||
|
||||
+5
-328
@@ -167,13 +167,6 @@ pub struct HealTaskStatus {
|
||||
/// Live progress snapshot; the exact shape is owned by the heal runtime.
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Canonical heal-owner result. Missing or future states are not repair proof.
|
||||
#[serde(default)]
|
||||
pub outcome: Option<serde_json::Value>,
|
||||
#[serde(default, alias = "next_seq")]
|
||||
pub next_seq: Option<u64>,
|
||||
#[serde(default, alias = "min_seq")]
|
||||
pub min_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// `POST /v3/background-heal/status` response. Known top-level fields are
|
||||
@@ -230,61 +223,6 @@ pub struct ScannerStatus {
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/scanner/cycle-state/reset` response for the legacy synchronous
|
||||
/// full-rescan reset path.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ScannerCycleResetResponse {
|
||||
pub status: String,
|
||||
pub mode: String,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/scanner/usage-state/reset` response for the legacy synchronous
|
||||
/// full-rebuild reset path.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ScannerUsageStateResetResponse {
|
||||
pub status: String,
|
||||
pub mode: String,
|
||||
pub usage_state: String,
|
||||
pub leader_epoch: u64,
|
||||
pub next_cycle: u64,
|
||||
pub reset_paths: Vec<String>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Durable scanner usage-state recovery intent response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ScannerUsageRecoveryIntentResponse {
|
||||
/// `accepted`, `replayed`, or `found`.
|
||||
pub status: String,
|
||||
pub action: String,
|
||||
pub mode: String,
|
||||
pub intent_id: String,
|
||||
pub state: String,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ScannerCycleResetRequest<'a> {
|
||||
mode: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ScannerUsageStateResetRequest<'a> {
|
||||
mode: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ScannerUsageStateAsyncResetRequest<'a> {
|
||||
mode: &'a str,
|
||||
#[serde(rename = "async")]
|
||||
async_intent: bool,
|
||||
idempotency_key: &'a str,
|
||||
}
|
||||
|
||||
/// Freshness block of the scanner status response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -420,22 +358,8 @@ impl AdminClient {
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
self.heal_status_since(bucket, prefix, client_token, None).await
|
||||
}
|
||||
|
||||
/// Query a retained result window. Missing cursors and outcome remain unknown.
|
||||
pub async fn heal_status_since(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
let mut query = vec![("clientToken", client_token.to_string())];
|
||||
if let Some(since_seq) = since_seq {
|
||||
query.push(("sinceSeq", since_seq.to_string()));
|
||||
}
|
||||
self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await
|
||||
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stop a heal: with a `client_token` only that task is cancelled and its
|
||||
@@ -454,7 +378,7 @@ impl AdminClient {
|
||||
match client_token {
|
||||
Some(_) => {
|
||||
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::Stopped(Box::new(status)))
|
||||
Ok(HealStopOutcome::Stopped(status))
|
||||
}
|
||||
None => {
|
||||
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
@@ -474,57 +398,6 @@ impl AdminClient {
|
||||
self.get_json("/v3/scanner/status").await
|
||||
}
|
||||
|
||||
/// Request a legacy synchronous scanner cycle reset (`full-rescan`).
|
||||
pub async fn scanner_cycle_state_reset_full_rescan(&self) -> Result<ScannerCycleResetResponse, AdminClientError> {
|
||||
let body =
|
||||
serde_json::to_vec(&ScannerCycleResetRequest { mode: "full-rescan" }).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
self.post_json("/v3/scanner/cycle-state/reset", &[], body).await
|
||||
}
|
||||
|
||||
/// Request a legacy synchronous scanner usage reset (`full-rebuild`).
|
||||
pub async fn scanner_usage_state_reset_full_rebuild(&self) -> Result<ScannerUsageStateResetResponse, AdminClientError> {
|
||||
let body = serde_json::to_vec(&ScannerUsageStateResetRequest { mode: "full-rebuild" }).map_err(|err| {
|
||||
AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
}
|
||||
})?;
|
||||
self.post_json("/v3/scanner/usage-state/reset", &[], body).await
|
||||
}
|
||||
|
||||
/// Accept a durable asynchronous scanner usage-state full-rebuild intent.
|
||||
///
|
||||
/// The caller owns `idempotency_key`; replaying the same key on the same
|
||||
/// server-side actor returns the same accepted intent instead of starting
|
||||
/// the legacy synchronous reset path.
|
||||
pub async fn scanner_usage_state_accept_full_rebuild_intent(
|
||||
&self,
|
||||
idempotency_key: &str,
|
||||
) -> Result<ScannerUsageRecoveryIntentResponse, AdminClientError> {
|
||||
let body = serde_json::to_vec(&ScannerUsageStateAsyncResetRequest {
|
||||
mode: "full-rebuild",
|
||||
async_intent: true,
|
||||
idempotency_key,
|
||||
})
|
||||
.map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
self.post_json("/v3/scanner/usage-state/reset", &[], body).await
|
||||
}
|
||||
|
||||
/// Query a durable asynchronous scanner usage-state recovery intent.
|
||||
pub async fn scanner_usage_state_recovery_intent_status(
|
||||
&self,
|
||||
intent_id: &str,
|
||||
) -> Result<ScannerUsageRecoveryIntentResponse, AdminClientError> {
|
||||
self.get_json(&format!(
|
||||
"/v3/scanner/usage-state/recovery-intents/{}",
|
||||
percent_encode_path_segment(intent_id)
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
/// ILM expiry worker status. The payload is owned by the expiry
|
||||
/// subsystem and still evolving; returned verbatim.
|
||||
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
@@ -660,7 +533,7 @@ impl AdminClient {
|
||||
/// start-success-shaped receipt.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealStopOutcome {
|
||||
Stopped(Box<HealTaskStatus>),
|
||||
Stopped(HealTaskStatus),
|
||||
PathStopped(HealStartSuccess),
|
||||
}
|
||||
|
||||
@@ -693,8 +566,7 @@ pub(crate) fn percent_encode_path_segment(segment: &str) -> String {
|
||||
mod tests {
|
||||
use super::{
|
||||
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
|
||||
ScannerCycleResetResponse, ScannerStatus, ScannerUsageRecoveryIntentResponse, ScannerUsageStateResetResponse, heal_path,
|
||||
percent_encode_path_segment,
|
||||
ScannerStatus, heal_path, percent_encode_path_segment,
|
||||
};
|
||||
use crate::test_support::TestServer;
|
||||
use serde_json::json;
|
||||
@@ -763,24 +635,6 @@ mod tests {
|
||||
assert!(status.progress.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_v3_decoder_preserves_canonical_unknown_and_future_fields() {
|
||||
let cases: serde_json::Value =
|
||||
serde_json::from_str(include_str!("../tests/fixtures/heal-outcome-v3.json")).expect("shared fixtures");
|
||||
for case in cases.as_array().expect("cases") {
|
||||
let status: HealTaskStatus = serde_json::from_value(case["response"].clone()).expect("optional outcome response");
|
||||
assert_eq!(status.outcome.as_ref(), Some(&case["response"]["outcome"]));
|
||||
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
|
||||
assert!(status.truncated);
|
||||
}
|
||||
let old: HealTaskStatus = serde_json::from_value(json!({"summary":"finished"})).expect("legacy response");
|
||||
assert!(old.outcome.is_none() && old.next_seq.is_none() && old.min_seq.is_none());
|
||||
let future = json!({"execution":{"state":"future_state"},"newField":7});
|
||||
let status: HealTaskStatus =
|
||||
serde_json::from_value(json!({"summary":"running","outcome":future})).expect("future outcome remains opaque");
|
||||
assert_eq!(status.outcome, Some(future));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
|
||||
let raw = json!({
|
||||
@@ -837,49 +691,6 @@ mod tests {
|
||||
assert_eq!(bare.freshness(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_reset_responses_preserve_future_fields() {
|
||||
let cycle: ScannerCycleResetResponse =
|
||||
serde_json::from_value(json!({"status": "reset", "mode": "full-rescan", "future": true})).unwrap();
|
||||
assert_eq!(cycle.status, "reset");
|
||||
assert_eq!(cycle.mode, "full-rescan");
|
||||
assert_eq!(cycle.extra["future"], true);
|
||||
|
||||
let usage: ScannerUsageStateResetResponse = serde_json::from_value(json!({
|
||||
"status": "reset",
|
||||
"mode": "full-rebuild",
|
||||
"usage_state": "bootstrap-pending",
|
||||
"leader_epoch": 11,
|
||||
"next_cycle": 42,
|
||||
"reset_paths": [".usage.json"],
|
||||
"future": {"accepted": false}
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(usage.status, "reset");
|
||||
assert_eq!(usage.mode, "full-rebuild");
|
||||
assert_eq!(usage.usage_state, "bootstrap-pending");
|
||||
assert_eq!(usage.leader_epoch, 11);
|
||||
assert_eq!(usage.next_cycle, 42);
|
||||
assert_eq!(usage.reset_paths, [".usage.json"]);
|
||||
assert_eq!(usage.extra["future"]["accepted"], false);
|
||||
|
||||
let intent: ScannerUsageRecoveryIntentResponse = serde_json::from_value(json!({
|
||||
"status": "accepted",
|
||||
"action": "usage-full-rebuild",
|
||||
"mode": "full-rebuild",
|
||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"state": "accepted",
|
||||
"future": {"worker": "pending"}
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(intent.status, "accepted");
|
||||
assert_eq!(intent.action, "usage-full-rebuild");
|
||||
assert_eq!(intent.mode, "full-rebuild");
|
||||
assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
|
||||
assert_eq!(intent.state, "accepted");
|
||||
assert_eq!(intent.extra["future"]["worker"], "pending");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_endpoint_is_rejected_without_io() {
|
||||
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
|
||||
@@ -922,101 +733,6 @@ mod tests {
|
||||
assert!(request.body.contains("\"recursive\":true"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_reset_posts_legacy_full_rescan_request() {
|
||||
let server = TestServer::spawn(r#"{"status":"reset","mode":"full-rescan"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let reset = client
|
||||
.scanner_cycle_state_reset_full_rescan()
|
||||
.await
|
||||
.expect("cycle reset response decodes");
|
||||
|
||||
assert_eq!(reset.status, "reset");
|
||||
assert_eq!(reset.mode, "full-rescan");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/cycle-state/reset");
|
||||
assert_eq!(request.query, "");
|
||||
assert!(request.body.contains("\"mode\":\"full-rescan\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_reset_posts_legacy_full_rebuild_request() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"reset","mode":"full-rebuild","usage_state":"bootstrap-pending","leader_epoch":11,"next_cycle":42,"reset_paths":[".usage.json"]}"#,
|
||||
200,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let reset = client
|
||||
.scanner_usage_state_reset_full_rebuild()
|
||||
.await
|
||||
.expect("usage reset response decodes");
|
||||
|
||||
assert_eq!(reset.status, "reset");
|
||||
assert_eq!(reset.mode, "full-rebuild");
|
||||
assert_eq!(reset.usage_state, "bootstrap-pending");
|
||||
assert_eq!(reset.leader_epoch, 11);
|
||||
assert_eq!(reset.next_cycle, 42);
|
||||
assert_eq!(reset.reset_paths, [".usage.json"]);
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||
assert_eq!(request.query, "");
|
||||
assert!(request.body.contains("\"mode\":\"full-rebuild\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_async_reset_posts_explicit_intent_contract() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted"}"#,
|
||||
202,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let accepted = client
|
||||
.scanner_usage_state_accept_full_rebuild_intent("intent-key-1")
|
||||
.await
|
||||
.expect("async recovery intent response decodes");
|
||||
|
||||
assert_eq!(accepted.status, "accepted");
|
||||
assert_eq!(accepted.mode, "full-rebuild");
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||
assert_eq!(request.query, "");
|
||||
assert!(request.body.contains("\"mode\":\"full-rebuild\""));
|
||||
assert!(request.body.contains("\"async\":true"));
|
||||
assert!(request.body.contains("\"idempotency_key\":\"intent-key-1\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_recovery_intent_status_gets_encoded_intent_id() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"found","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"id%2Fwith%20space","state":"accepted"}"#,
|
||||
200,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let status = client
|
||||
.scanner_usage_state_recovery_intent_status("id/with space")
|
||||
.await
|
||||
.expect("recovery intent status response decodes");
|
||||
|
||||
assert_eq!(status.status, "found");
|
||||
assert_eq!(status.action, "usage-full-rebuild");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "GET");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/recovery-intents/id%2Fwith%20space");
|
||||
assert_eq!(request.query, "");
|
||||
assert_eq!(request.body, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_sends_client_token_on_the_same_path() {
|
||||
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
@@ -1034,26 +750,6 @@ mod tests {
|
||||
assert!(!request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outcome_v3_since_query_preserves_cursor_and_never_sends_force_start() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"summary":"running","nextSeq":9,"minSeq":4,"truncated":true,"outcome":{"execution":{"state":"future_state"}}}"#,
|
||||
200,
|
||||
)
|
||||
.await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("test client");
|
||||
let status = client
|
||||
.heal_status_since(Some("bucket"), None, "token-1", Some(3))
|
||||
.await
|
||||
.expect("window response");
|
||||
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
|
||||
assert!(status.truncated);
|
||||
assert_eq!(status.outcome.expect("future state is preserved")["execution"]["state"], "future_state");
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("sinceSeq=3") && request.query.contains("clientToken=token-1"));
|
||||
assert!(!request.query.contains("forceStart") && !request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_without_token_takes_the_path_cancel_branch() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
|
||||
@@ -1066,25 +762,6 @@ mod tests {
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_with_token_decodes_boxed_task_status() {
|
||||
let body = r#"{"summary":"stopped","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let outcome = client
|
||||
.heal_stop(Some("bucket"), None, Some("token-1"))
|
||||
.await
|
||||
.expect("token stop decodes");
|
||||
let super::HealStopOutcome::Stopped(status) = outcome else {
|
||||
panic!("token stop should return task status");
|
||||
};
|
||||
assert_eq!(status.summary, "stopped");
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("forceStop=true"));
|
||||
assert!(request.query.contains("clientToken=token-1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_heal_status_posts_to_the_registered_route() {
|
||||
let body = r#"{"state":"idle","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":true}"#;
|
||||
|
||||
-472
@@ -1,472 +0,0 @@
|
||||
[
|
||||
{
|
||||
"name": "completed",
|
||||
"cliExit": 0,
|
||||
"response": {
|
||||
"summary": "finished",
|
||||
"detail": "heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "unknown",
|
||||
"cliExit": 0,
|
||||
"response": {
|
||||
"summary": "finished",
|
||||
"detail": "heal traversal completed; authoritative storage proof is unavailable for 1 objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 1,
|
||||
"failed": 0,
|
||||
"unknown": 1,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "unknown"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "completed_with_errors",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors"
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "cancelled",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task cancelled; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "deadline",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task timed out; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "deadline"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "untraversable",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal listing is untraversable; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "untraversable"
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "remote_completed_with_errors",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
},
|
||||
"remoteResponse": {
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "completed_with_errors",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "complete",
|
||||
"counters": {
|
||||
"processed": 1,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 1,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 1,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [
|
||||
{
|
||||
"identity": {
|
||||
"kind": "object",
|
||||
"bucket": "bucket",
|
||||
"object": "object",
|
||||
"versionId": null,
|
||||
"bucketIncarnationId": null,
|
||||
"poolIndex": null,
|
||||
"setIndex": null
|
||||
},
|
||||
"disposition": {
|
||||
"state": "failed",
|
||||
"details": "retry_exhausted"
|
||||
},
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "remote_cancelled",
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal task cancelled; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
},
|
||||
"remoteResponse": {
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
"scanMode": 1
|
||||
},
|
||||
"items": [],
|
||||
"truncated": true,
|
||||
"nextSeq": 9,
|
||||
"minSeq": 4,
|
||||
"progress": {
|
||||
"objectsScanned": 11,
|
||||
"objectsHealed": 7
|
||||
},
|
||||
"outcome": {
|
||||
"execution": {
|
||||
"state": "aborted",
|
||||
"reason": "cancelled",
|
||||
"futureExtension": {
|
||||
"value": 7
|
||||
}
|
||||
},
|
||||
"coverage": "partial",
|
||||
"counters": {
|
||||
"processed": 0,
|
||||
"healed": 0,
|
||||
"unchanged": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
"attemptFailures": 0,
|
||||
"overflowed": false,
|
||||
"futureCounter": 11
|
||||
},
|
||||
"objects": [],
|
||||
"objectsTruncated": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1257,8 +1257,6 @@ pub struct ScannerDirtyUsageBucket {
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub generation: u64,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub bucket_incarnation: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -1284,8 +1282,6 @@ pub struct ScannerDirtyUsageSnapshotResponse {
|
||||
pub buckets: ::prost::alloc::vec::Vec<ScannerDirtyUsageBucket>,
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
#[prost(string, tag = "8")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
}
|
||||
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
/// have a durable per-bucket publication proof.
|
||||
|
||||
@@ -175,9 +175,6 @@ pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0";
|
||||
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
|
||||
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
|
||||
pub const ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-ilm-recovery-export-capability-v1\0";
|
||||
pub const TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX: &[u8] =
|
||||
b"rustfs-transition-transaction-compaction-capability-v1\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
@@ -222,30 +219,6 @@ pub fn is_cross_pool_fence_capability_probe(command: &[u8]) -> bool {
|
||||
&& command.starts_with(CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn ilm_recovery_export_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_ilm_recovery_export_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn transition_transaction_compaction_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_transition_transaction_compaction_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn encode_remote_version_state_capability(
|
||||
topology_member: &str,
|
||||
process_epoch: &[u8; 16],
|
||||
@@ -589,13 +562,11 @@ pub fn canonical_scanner_dirty_usage_snapshot_response_body(
|
||||
body.push_u64(response.generation);
|
||||
body.push_u64(response.pending_bucket_count);
|
||||
body.push_u32(response.protocol_version);
|
||||
body.push_str(&response.owner_id)?;
|
||||
body.push_bool(response.complete);
|
||||
body.push_count(response.buckets.len())?;
|
||||
for bucket in &response.buckets {
|
||||
body.push_str(&bucket.bucket)?;
|
||||
body.push_u64(bucket.generation);
|
||||
body.push_bytes(bucket.bucket_incarnation.as_ref())?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
@@ -1858,16 +1829,13 @@ mod scanner_activity_tests {
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "archive".to_string(),
|
||||
generation: 3,
|
||||
bucket_incarnation: vec![1; 16].into(),
|
||||
},
|
||||
ScannerDirtyUsageBucket {
|
||||
bucket: "photos".to_string(),
|
||||
generation: 7,
|
||||
bucket_incarnation: vec![2; 16].into(),
|
||||
},
|
||||
],
|
||||
response_proof: vec![9; 32].into(),
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
};
|
||||
let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response)
|
||||
.expect("scanner dirty usage snapshot response should encode");
|
||||
@@ -1884,9 +1852,6 @@ mod scanner_activity_tests {
|
||||
let mut protocol = response.clone();
|
||||
protocol.protocol_version = 2;
|
||||
variants.push(protocol);
|
||||
let mut owner = response.clone();
|
||||
owner.owner_id = "22222222-2222-2222-2222-222222222222".to_string();
|
||||
variants.push(owner);
|
||||
let mut complete = response.clone();
|
||||
complete.complete = false;
|
||||
variants.push(complete);
|
||||
@@ -1896,9 +1861,6 @@ mod scanner_activity_tests {
|
||||
let mut bucket_generation = response.clone();
|
||||
bucket_generation.buckets[0].generation = 4;
|
||||
variants.push(bucket_generation);
|
||||
let mut bucket_incarnation = response.clone();
|
||||
bucket_incarnation.buckets[0].bucket_incarnation = vec![3; 16].into();
|
||||
variants.push(bucket_incarnation);
|
||||
let mut bucket_order = response.clone();
|
||||
bucket_order.buckets.reverse();
|
||||
variants.push(bucket_order);
|
||||
@@ -2165,15 +2127,12 @@ mod scanner_activity_tests {
|
||||
mod heal_control_tests {
|
||||
use super::{
|
||||
CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
|
||||
TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack,
|
||||
canonical_heal_control_request_body, canonical_heal_control_response_body, decode_remote_version_state_capability,
|
||||
encode_cross_pool_fence_capability, encode_remote_version_state_capability, heal_control_capability_probe,
|
||||
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
|
||||
ilm_recovery_export_capability_probe, internode_rpc_timeout, is_cross_pool_fence_capability_probe,
|
||||
is_heal_control_capability_probe, is_ilm_recovery_export_capability_probe, is_remote_version_state_capability_probe,
|
||||
is_transition_transaction_compaction_capability_probe, normalize_internode_rpc_timeout,
|
||||
remote_version_state_capability_probe, transition_transaction_compaction_capability_probe,
|
||||
REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack, canonical_heal_control_request_body,
|
||||
canonical_heal_control_response_body, decode_remote_version_state_capability, encode_cross_pool_fence_capability,
|
||||
encode_remote_version_state_capability, heal_control_capability_probe, heal_control_coordinator_epoch,
|
||||
heal_control_execution_timeout, heal_control_execution_timeout_for, internode_rpc_timeout,
|
||||
is_cross_pool_fence_capability_probe, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
|
||||
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
|
||||
};
|
||||
use crate::heal_control;
|
||||
use std::time::Duration;
|
||||
@@ -2237,34 +2196,6 @@ mod heal_control_tests {
|
||||
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ilm_recovery_export_capability_probe_requires_exact_prefix_and_nonce() {
|
||||
let probe = ilm_recovery_export_capability_probe(&[7; 16]);
|
||||
assert!(is_ilm_recovery_export_capability_probe(&probe));
|
||||
assert!(!is_ilm_recovery_export_capability_probe(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX));
|
||||
let mut wrong_prefix = probe.clone();
|
||||
wrong_prefix[0] ^= 1;
|
||||
assert!(!is_ilm_recovery_export_capability_probe(&wrong_prefix));
|
||||
let mut extra = probe;
|
||||
extra.push(0);
|
||||
assert!(!is_ilm_recovery_export_capability_probe(&extra));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_transaction_compaction_probe_requires_exact_prefix_and_nonce() {
|
||||
let probe = transition_transaction_compaction_capability_probe(&[7; 16]);
|
||||
assert!(is_transition_transaction_compaction_capability_probe(&probe));
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(
|
||||
TRANSITION_TRANSACTION_COMPACTION_CAPABILITY_PROBE_PREFIX,
|
||||
));
|
||||
let mut wrong_prefix = probe.clone();
|
||||
wrong_prefix[0] ^= 1;
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(&wrong_prefix));
|
||||
let mut extra = probe;
|
||||
extra.push(0);
|
||||
assert!(!is_transition_transaction_compaction_capability_probe(&extra));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_binds_member_and_process_epoch() {
|
||||
let encoded =
|
||||
|
||||
@@ -885,7 +885,6 @@ message ScannerActivityResponse {
|
||||
message ScannerDirtyUsageBucket {
|
||||
string bucket = 1;
|
||||
uint64 generation = 2;
|
||||
bytes bucket_incarnation = 3;
|
||||
}
|
||||
|
||||
message ScannerDirtyUsageSnapshotRequest {
|
||||
@@ -902,7 +901,6 @@ message ScannerDirtyUsageSnapshotResponse {
|
||||
bool complete = 5;
|
||||
repeated ScannerDirtyUsageBucket buckets = 6;
|
||||
bytes response_proof = 7;
|
||||
string owner_id = 8;
|
||||
}
|
||||
|
||||
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
|
||||
@@ -122,10 +122,6 @@ pub trait ReplicationConfigurationExt {
|
||||
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String>;
|
||||
/// Every target ARN the configuration still names, whatever the rule's
|
||||
/// status, prefix or filter: the set a pending replication delete may
|
||||
/// still be owed to. A target outside it was removed by the operator.
|
||||
fn configured_target_arns(&self) -> HashSet<String>;
|
||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||
self.filter_target_arns(obj)
|
||||
.into_iter()
|
||||
@@ -776,19 +772,6 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
}
|
||||
|
||||
/// Filter target ARNs and return a slice of the distinct values in the config
|
||||
fn configured_target_arns(&self) -> HashSet<String> {
|
||||
let role = self.role.trim();
|
||||
if !role.is_empty() {
|
||||
return HashSet::from([role.to_string()]);
|
||||
}
|
||||
self.rules
|
||||
.iter()
|
||||
.map(|rule| rule.destination.bucket.trim())
|
||||
.filter(|arn| !arn.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
|
||||
let role = self.role.trim();
|
||||
if !role.is_empty() {
|
||||
|
||||
@@ -438,12 +438,6 @@ pub struct ReplicatedTargetInfo {
|
||||
/// Version the target assigned to the delete marker it just created.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target_delete_marker_version_id: Option<String>,
|
||||
/// Version the target assigned to this object version when it differs
|
||||
/// from the source id (a target that mints its own ids). Persisted as the
|
||||
/// per-target ledger every later version-addressed mutation resolves
|
||||
/// through; `None` on targets that adopt the source id.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target_version_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ReplicatedTargetInfo {
|
||||
|
||||
@@ -65,10 +65,9 @@ pub use multipart::{
|
||||
};
|
||||
pub use object::{
|
||||
ObjectLockIntegrity, ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
|
||||
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity,
|
||||
replication_action_for_target, replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present,
|
||||
ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity, replication_action_for_target,
|
||||
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
|
||||
@@ -234,62 +234,18 @@ fn comparable_metadata(metadata: Option<&HashMap<String, String>>) -> HashMap<St
|
||||
/// real (non-nil) version uuid, and drift means the target answered with
|
||||
/// anything else — including nothing at all.
|
||||
pub fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
|
||||
version_identity_capability_from_put(source_version_id, assigned_version_id) == Some(VersionIdentityCapability::MintsOwn)
|
||||
}
|
||||
|
||||
/// Whether a replication target adopts the source version id it is handed on
|
||||
/// PutObject / CompleteMultipartUpload, or mints its own.
|
||||
///
|
||||
/// A target that mints its own ids (AWS S3, Wasabi, Impossible Cloud) still
|
||||
/// stores the bytes, but every later version-addressed request from the
|
||||
/// source names an id the target never had. Its HEAD then answers 404 —
|
||||
/// indistinguishable from a replica that is really missing — so a heal, MRF
|
||||
/// retry or existing-object resync re-drive would PUT the object again and
|
||||
/// mint yet another target version (rustfs/backlog#2340). The replication
|
||||
/// worker learns the verdict from each PUT response (and replication-check's
|
||||
/// VersionFidelity phase) and, once `MintsOwn` is known, locates a replica by
|
||||
/// exact key and ETag before concluding that it is missing. The verdict cache
|
||||
/// is owned by the runtime's bucket target system; this crate owns only the
|
||||
/// vocabulary and the judgment.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum VersionIdentityCapability {
|
||||
#[default]
|
||||
Unknown,
|
||||
Adopts,
|
||||
MintsOwn,
|
||||
}
|
||||
|
||||
impl VersionIdentityCapability {
|
||||
/// True when a 404 from a version-addressed HEAD on this target cannot be
|
||||
/// read as "replica missing": the source-side id was never the target's.
|
||||
pub fn version_addressing_unreliable(self) -> bool {
|
||||
self == VersionIdentityCapability::MintsOwn
|
||||
}
|
||||
}
|
||||
|
||||
/// Judge the identity contract from one replication write: `None` when no
|
||||
/// contract applies (the source addressed no real version — an empty or nil
|
||||
/// uuid travels as the literal "null", unversioned-source semantics),
|
||||
/// otherwise whether the target echoed the source id or answered with
|
||||
/// anything else — including nothing at all.
|
||||
pub fn version_identity_capability_from_put(
|
||||
source_version_id: &str,
|
||||
assigned_version_id: Option<&str>,
|
||||
) -> Option<VersionIdentityCapability> {
|
||||
if source_version_id.is_empty() {
|
||||
return None;
|
||||
return false;
|
||||
}
|
||||
// A nil source uuid travels as the literal "null" (unversioned-source
|
||||
// semantics); no identity contract applies to it.
|
||||
if uuid::Uuid::parse_str(source_version_id)
|
||||
.map(|uuid| uuid.is_nil())
|
||||
.unwrap_or(true)
|
||||
{
|
||||
return None;
|
||||
return false;
|
||||
}
|
||||
Some(if assigned_version_id == Some(source_version_id) {
|
||||
VersionIdentityCapability::Adopts
|
||||
} else {
|
||||
VersionIdentityCapability::MintsOwn
|
||||
})
|
||||
assigned_version_id != Some(source_version_id)
|
||||
}
|
||||
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
@@ -421,9 +377,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
|
||||
VersionIdentityCapability, content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target,
|
||||
replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_capability_from_put, version_identity_drifted,
|
||||
content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
|
||||
single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
};
|
||||
use crate::filemeta::{ReplicationAction, ReplicationType};
|
||||
use crate::http::AMZ_OBJECT_LOCK_MODE;
|
||||
@@ -552,32 +508,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_capability_is_judged_only_for_real_source_versions() {
|
||||
let source = "8e4d2f4c-2d5c-4f1b-9d0a-9c8b7a6f5e4d";
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, Some(source)),
|
||||
Some(VersionIdentityCapability::Adopts)
|
||||
);
|
||||
// Wasabi / AWS shape: a minted id, or no id at all, both mean the
|
||||
// source-side id is not addressable on the target.
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, Some("001788697733811332140-fR6j6uXKV-")),
|
||||
Some(VersionIdentityCapability::MintsOwn)
|
||||
);
|
||||
assert_eq!(
|
||||
version_identity_capability_from_put(source, None),
|
||||
Some(VersionIdentityCapability::MintsOwn)
|
||||
);
|
||||
// No contract for an unversioned source write.
|
||||
assert_eq!(version_identity_capability_from_put("", Some("anything")), None);
|
||||
assert_eq!(version_identity_capability_from_put("00000000-0000-0000-0000-000000000000", None), None);
|
||||
assert_eq!(version_identity_capability_from_put("null", Some("null")), None);
|
||||
assert!(VersionIdentityCapability::MintsOwn.version_addressing_unreliable());
|
||||
assert!(!VersionIdentityCapability::Adopts.version_addressing_unreliable());
|
||||
assert!(!VersionIdentityCapability::Unknown.version_addressing_unreliable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_offline_error_classifier_is_network_scoped() {
|
||||
assert!(is_replication_target_offline_error("put_object dispatch failure: connector error"));
|
||||
|
||||
@@ -106,7 +106,6 @@ bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-heal.workspace = true
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -71,8 +71,6 @@ const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
|
||||
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
|
||||
|
||||
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
|
||||
pub const DATA_USAGE_RAW_ENUMERATION_CURSOR_VERSION: u16 = 1;
|
||||
const DATA_USAGE_SCAN_CURSOR_MAX_BYTES: usize = 16 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum DataUsageCacheRevision {
|
||||
@@ -403,54 +401,6 @@ impl DataUsageScanCheckpoint {
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable raw directory-page cursor for a bucket scan.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DataUsageRawEnumerationCursor {
|
||||
pub version: u16,
|
||||
pub parent: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_entry: Option<String>,
|
||||
pub entries_seen: u64,
|
||||
pub page_digest: [u8; 32],
|
||||
}
|
||||
|
||||
impl DataUsageRawEnumerationCursor {
|
||||
pub fn new(parent: String, last_entry: Option<String>, entries_seen: u64, page_digest: [u8; 32]) -> Self {
|
||||
Self {
|
||||
version: DATA_USAGE_RAW_ENUMERATION_CURSOR_VERSION,
|
||||
parent,
|
||||
last_entry,
|
||||
entries_seen,
|
||||
page_digest,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_for_bucket(&self, bucket: &str) -> bool {
|
||||
self.version == DATA_USAGE_RAW_ENUMERATION_CURSOR_VERSION
|
||||
&& bucket != DATA_USAGE_ROOT
|
||||
&& path_is_in_bucket_scope(bucket, &self.parent)
|
||||
&& self.parent.len() <= DATA_USAGE_SCAN_CURSOR_MAX_BYTES
|
||||
&& self.page_digest != [0; 32]
|
||||
&& match &self.last_entry {
|
||||
Some(last_entry) => {
|
||||
!last_entry.is_empty()
|
||||
&& self.entries_seen > 0
|
||||
&& last_entry.len() <= DATA_USAGE_SCAN_CURSOR_MAX_BYTES
|
||||
&& !last_entry.contains(SLASH_SEPARATOR)
|
||||
}
|
||||
None => self.entries_seen == 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn path_is_in_bucket_scope(bucket: &str, path: &str) -> bool {
|
||||
path == bucket
|
||||
|| path
|
||||
.strip_prefix(bucket)
|
||||
.is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
|
||||
}
|
||||
|
||||
/// Durable scope of a bucket checkpoint, independent of namespace mutation counters.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -600,8 +550,6 @@ pub struct DataUsageCacheInfo {
|
||||
#[serde(default)]
|
||||
pub scan_checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
#[serde(default)]
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default)]
|
||||
pub scan_identity: Option<DataUsageScanIdentity>,
|
||||
#[serde(default)]
|
||||
pub scan_progress: Option<DataUsageScanProgress>,
|
||||
@@ -660,7 +608,6 @@ impl Serialize for DataUsageCacheInfo {
|
||||
// Keep this metadata map-encoded so older readers can ignore fields
|
||||
// appended by newer scanner versions during rolling upgrades.
|
||||
let field_count = 16
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_coverage_receipt.is_some())
|
||||
@@ -684,9 +631,6 @@ impl Serialize for DataUsageCacheInfo {
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("scan_resume_after", &self.scan_resume_after)?;
|
||||
state.serialize_entry("scan_checkpoint", &self.scan_checkpoint)?;
|
||||
if let Some(cursor) = &self.scan_raw_enumeration_cursor {
|
||||
state.serialize_entry("scan_raw_enumeration_cursor", cursor)?;
|
||||
}
|
||||
if let Some(identity) = self.scan_identity {
|
||||
state.serialize_entry("scan_identity", &identity)?;
|
||||
}
|
||||
@@ -894,7 +838,6 @@ impl DataUsageCache {
|
||||
&& self.info.snapshot_complete
|
||||
&& self.info.scan_progress.is_none()
|
||||
&& self.info.scan_checkpoint.is_none()
|
||||
&& self.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& self.info.scan_resume_after.is_none()
|
||||
&& self.info.scan_coverage_receipt.is_none()
|
||||
&& self.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
@@ -919,15 +862,10 @@ impl DataUsageCache {
|
||||
self.info.pending_heals = pending_heals;
|
||||
self.info.size_reconciliation = size_reconciliation;
|
||||
}
|
||||
if self.validated_raw_enumeration_cursor().is_none() {
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
}
|
||||
let cursor_is_valid = (self.info.scan_checkpoint.is_none()
|
||||
&& self.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& self.info.scan_resume_after.is_none()
|
||||
&& self.info.scan_coverage_receipt.is_none())
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some();
|
||||
|| self.validated_scan_frontier().is_some();
|
||||
if !cursor_is_valid {
|
||||
self.info.scan_progress = None;
|
||||
}
|
||||
@@ -948,7 +886,6 @@ impl DataUsageCache {
|
||||
});
|
||||
self.info.scan_resume_after = None;
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -1017,15 +954,6 @@ impl DataUsageCache {
|
||||
.then_some(receipt.through.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn validated_raw_enumeration_cursor(&self) -> Option<&DataUsageRawEnumerationCursor> {
|
||||
let cursor = self.info.scan_raw_enumeration_cursor.as_ref()?;
|
||||
(self.info.scan_progress.is_some()
|
||||
&& self.info.scan_identity.is_some_and(|identity| identity.is_valid())
|
||||
&& self.info.source.is_some()
|
||||
&& cursor.is_valid_for_bucket(&self.info.name))
|
||||
.then_some(cursor)
|
||||
}
|
||||
|
||||
/// Seal only the frontier supplied by completed traversal, never a restored cursor.
|
||||
pub(crate) fn seal_scan_frontier(&mut self, frontier: Option<&str>) -> Result<(), serde_json::Error> {
|
||||
if self.info.scan_progress.is_none() {
|
||||
|
||||
@@ -1133,7 +1133,6 @@ fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after()
|
||||
assert_eq!(decoded.failed_objects.get("bad-object"), Some(&11));
|
||||
assert!(decoded.scan_resume_after.is_none());
|
||||
assert!(decoded.scan_checkpoint.is_none());
|
||||
assert!(decoded.scan_raw_enumeration_cursor.is_none());
|
||||
assert!(decoded.pending_heals.is_empty());
|
||||
assert!(decoded.source.is_none());
|
||||
assert!(!decoded.snapshot_complete);
|
||||
@@ -1173,12 +1172,6 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
skip_healing: true,
|
||||
failed_objects: HashMap::from([("bad-object".to_string(), 11)]),
|
||||
source: Some(DataUsageCacheSource::new(1, 2)),
|
||||
scan_raw_enumeration_cursor: Some(DataUsageRawEnumerationCursor::new(
|
||||
"bucket/prefix".to_string(),
|
||||
Some("last-object".to_string()),
|
||||
7,
|
||||
[7; 32],
|
||||
)),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])),
|
||||
@@ -1199,14 +1192,6 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
let current = DataUsageCache::unmarshal(&buf).expect("Current reader failed to deserialize new cache");
|
||||
assert_eq!(current.info.leader_epoch, 9);
|
||||
assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
|
||||
assert_eq!(
|
||||
current
|
||||
.info
|
||||
.scan_raw_enumeration_cursor
|
||||
.as_ref()
|
||||
.map(|cursor| cursor.last_entry.as_deref()),
|
||||
Some(Some("last-object"))
|
||||
);
|
||||
assert!(current.info.snapshot_complete);
|
||||
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
|
||||
assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32])));
|
||||
@@ -1229,123 +1214,6 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
assert_eq!(decoded.cache.get("bucket").map(|entry| entry.objects), Some(3));
|
||||
}
|
||||
|
||||
fn valid_scan_identity() -> DataUsageScanIdentity {
|
||||
DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: uuid::Uuid::new_v4(),
|
||||
set_layout: TEST_PLAN_DIGEST,
|
||||
publication_epoch: 5,
|
||||
tier_registry_generation: 9,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_with_raw_cursor(cursor: DataUsageRawEnumerationCursor) -> DataUsageCache {
|
||||
DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
leader_epoch: 1,
|
||||
source: Some(DataUsageCacheSource::new(1, 2)),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
scan_identity: Some(valid_scan_identity()),
|
||||
tier_registry_generation: Some(9),
|
||||
scan_progress: Some(DataUsageScanProgress {
|
||||
started_plan: TEST_PLAN_DIGEST,
|
||||
requested_plan: TEST_PLAN_DIGEST,
|
||||
}),
|
||||
scan_raw_enumeration_cursor: Some(cursor),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_cursor_validation_requires_bucket_identity_and_bounded_marker() {
|
||||
let valid = DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), Some("entry-001".to_string()), 1, [8; 32]);
|
||||
let cache = cache_with_raw_cursor(valid.clone());
|
||||
assert_eq!(cache.validated_raw_enumeration_cursor(), Some(&valid));
|
||||
|
||||
let page_begin = DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), None, 0, [8; 32]);
|
||||
let cache = cache_with_raw_cursor(page_begin.clone());
|
||||
assert_eq!(cache.validated_raw_enumeration_cursor(), Some(&page_begin));
|
||||
|
||||
let seen_without_marker = DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), None, 1, [8; 32]);
|
||||
assert!(
|
||||
cache_with_raw_cursor(seen_without_marker)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let mut missing_identity = cache_with_raw_cursor(valid.clone());
|
||||
missing_identity.info.scan_identity = None;
|
||||
assert!(missing_identity.validated_raw_enumeration_cursor().is_none());
|
||||
|
||||
let mut outside_bucket = valid.clone();
|
||||
outside_bucket.parent = "other/raw".to_string();
|
||||
assert!(
|
||||
cache_with_raw_cursor(outside_bucket)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let mut future_version = valid.clone();
|
||||
future_version.version = DATA_USAGE_RAW_ENUMERATION_CURSOR_VERSION + 1;
|
||||
assert!(
|
||||
cache_with_raw_cursor(future_version)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let mut zero_digest = valid.clone();
|
||||
zero_digest.page_digest = [0; 32];
|
||||
assert!(
|
||||
cache_with_raw_cursor(zero_digest)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let mut nested_marker = valid;
|
||||
nested_marker.last_entry = Some("child/object".to_string());
|
||||
assert!(
|
||||
cache_with_raw_cursor(nested_marker)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let oversized_marker =
|
||||
DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), Some("x".repeat(16 * 1024 + 1)), 1, [8; 32]);
|
||||
assert!(
|
||||
cache_with_raw_cursor(oversized_marker)
|
||||
.validated_raw_enumeration_cursor()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_bucket_checkpoint_preserves_only_valid_raw_enumeration_cursor() {
|
||||
let identity = valid_scan_identity();
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let cursor = DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), Some("entry-001".to_string()), 1, [9; 32]);
|
||||
let mut cache = cache_with_raw_cursor(cursor.clone());
|
||||
cache.info.scan_identity = Some(identity);
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 1, 1, source, TEST_PLAN_DIGEST, identity),
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert_eq!(cache.info.scan_raw_enumeration_cursor, Some(cursor));
|
||||
|
||||
let invalid = DataUsageRawEnumerationCursor::new("other/raw".to_string(), Some("entry-001".to_string()), 1, [9; 32]);
|
||||
let mut cache = cache_with_raw_cursor(invalid);
|
||||
cache.info.scan_identity = Some(identity);
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 1, 1, source, TEST_PLAN_DIGEST, identity),
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert!(cache.info.scan_raw_enumeration_cursor.is_none());
|
||||
assert!(cache.info.scan_progress.is_some());
|
||||
}
|
||||
|
||||
/// Deterministic, fully populated cache used to pin the persisted
|
||||
/// `.usage-cache.bin` wire bytes. Every map/set holds at most one element
|
||||
/// so the map-encoded `marshal_msg` output is byte-stable.
|
||||
|
||||
@@ -83,19 +83,16 @@ pub use remote_scanner::{
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_scanner_metrics::last_minute;
|
||||
pub use scanner::{
|
||||
SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD, ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus,
|
||||
ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus,
|
||||
ScannerPauseBacklogThresholds, ScannerRecoveryIntentAcceptResult, ScannerRecoveryIntentConflict, ScannerRecoveryIntentRecord,
|
||||
ScannerRecoveryIntentRequest, ScannerUsageStateResetResult, accept_scanner_usage_recovery_intent,
|
||||
get_scanner_usage_recovery_intent, init_data_scanner, init_scanner_with_recovery, reset_scanner_cycle_recovery,
|
||||
reset_scanner_usage_state_for_full_rebuild, run_scanner_usage_recovery_intent, scanner_cycle_recovery_status,
|
||||
scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_recovery_actor_sha256, scanner_topology_digest,
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
|
||||
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
|
||||
init_data_scanner, reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
|
||||
scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
@@ -103,7 +100,6 @@ pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
|
||||
|
||||
|
||||
@@ -16,9 +16,8 @@
|
||||
use crate::RUSTFS_META_BUCKET;
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
|
||||
use crate::scanner_io::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk,
|
||||
acquire_scanner_cache_locks, cache_root_entry_info, current_cache_root_or_prepare_with_generation,
|
||||
scanner_set_disk_inventory,
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks,
|
||||
cache_root_entry_info, current_cache_root_or_prepare_with_generation, scanner_set_disk_inventory,
|
||||
};
|
||||
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::{
|
||||
@@ -780,18 +779,7 @@ async fn scan_and_persist_local_bucket(
|
||||
|
||||
let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
|
||||
let scan_ctx = ctx.child_token();
|
||||
let scan = ScannerIODisk::nsscanner_disk(
|
||||
disk.clone(),
|
||||
scan_ctx.clone(),
|
||||
budget,
|
||||
set_disks,
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
},
|
||||
);
|
||||
let scan = ScannerIODisk::nsscanner_disk(disk.clone(), scan_ctx.clone(), budget, set_disks, cache, None, scan_mode);
|
||||
tokio::pin!(scan);
|
||||
let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL);
|
||||
tokio::pin!(fence_watch);
|
||||
|
||||
+36
-175
@@ -14,6 +14,7 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
@@ -70,13 +71,12 @@ use crate::storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
};
|
||||
use crate::{
|
||||
DiskError, ECStore, EcstoreError, ListPathRawOptions, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED,
|
||||
ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch,
|
||||
get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, list_path_raw, read_config, replace_bucket_usage_memory_from_info, save_config,
|
||||
save_config_shared_with_preconditions_and_lease_fence_and_scope, save_config_with_preconditions,
|
||||
save_config_with_publication_admission_for_epoch, scanner_publication_admission_for_epoch, scanner_publication_epoch,
|
||||
scanner_publication_epoch_changed,
|
||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
|
||||
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
|
||||
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
|
||||
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence_and_scope,
|
||||
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_publication_admission_for_epoch,
|
||||
scanner_publication_epoch, scanner_publication_epoch_changed,
|
||||
};
|
||||
|
||||
const LOG_COMPONENT_SCANNER: &str = "scanner";
|
||||
@@ -948,71 +948,6 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
init_data_scanner_with_storage(ctx, storeapi).await;
|
||||
}
|
||||
|
||||
async fn run_scanner_usage_recovery_intents_for_startup(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
) -> Result<usize, ScannerError> {
|
||||
let intent_ids = scanner_usage_recovery_intents_for_startup(&ctx, storeapi.clone()).await?;
|
||||
let mut attempted = 0usize;
|
||||
for intent_id in intent_ids {
|
||||
if ctx.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
run_scanner_usage_recovery_intent(ctx.child_token(), storeapi.clone(), intent_id).await?;
|
||||
attempted = attempted.saturating_add(1);
|
||||
}
|
||||
Ok(attempted)
|
||||
}
|
||||
|
||||
/// Start normal scanning when enabled, or one resume-only cleanup attempt.
|
||||
/// The disabled branch returns a finite task for the startup owner to join;
|
||||
/// it never enables ordinary namespace scanning or accepts a new reset intent.
|
||||
pub async fn init_scanner_with_recovery(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
enabled: bool,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
if enabled {
|
||||
if let Err(error) = run_scanner_usage_recovery_intents_for_startup(ctx.clone(), storeapi.clone()).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "recovery_intent_startup_discovery_failed",
|
||||
error = %error,
|
||||
"Scanner recovery intent startup discovery failed"
|
||||
);
|
||||
}
|
||||
init_data_scanner(ctx, storeapi).await;
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(error) = run_scanner_usage_recovery_intents_for_startup(ctx.clone(), storeapi.clone()).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "recovery_intent_startup_discovery_failed",
|
||||
error = %error,
|
||||
"Disabled scanner recovery intent startup discovery failed"
|
||||
);
|
||||
}
|
||||
if let Err(error) = resume_scanner_cycle_cleanup(ctx, storeapi).await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "disabled_cleanup_deferred",
|
||||
error = %error,
|
||||
"Disabled scanner cleanup remains pending for an operator retry"
|
||||
);
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn init_data_scanner_with_storage<S>(ctx: CancellationToken, storeapi: Arc<S>)
|
||||
where
|
||||
S: ScannerStorage,
|
||||
@@ -1623,11 +1558,6 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
}
|
||||
|
||||
struct ScannerCycleScheduling {
|
||||
requires_full_scan: bool,
|
||||
service_cohort: Option<Arc<StdMutex<crate::scanner_io::ScannerServiceCohort>>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn run_data_scanner_cycle<S>(
|
||||
ctx: &CancellationToken,
|
||||
@@ -1640,19 +1570,7 @@ where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
|
||||
run_data_scanner_cycle_with_budget(
|
||||
ctx,
|
||||
storeapi,
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget,
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget, true).await
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
@@ -1664,7 +1582,7 @@ async fn run_data_scanner_cycle_with_budget<S>(
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_budget: Arc<ScannerCycleBudget>,
|
||||
scheduling: ScannerCycleScheduling,
|
||||
requires_full_scan: bool,
|
||||
) -> ScannerCycleOutcome
|
||||
where
|
||||
S: ScannerStorage,
|
||||
@@ -1769,7 +1687,6 @@ where
|
||||
};
|
||||
let baseline_publication_epoch = baseline_publication_guard.epoch();
|
||||
let usage_persist_baseline_result = read_data_usage_persist_baseline(storeapi.clone()).await;
|
||||
let observed_usage_candidate_result = read_config(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
|
||||
drop(baseline_publication_guard);
|
||||
let usage_persist_baseline = match usage_persist_baseline_result {
|
||||
Ok(baseline) => baseline,
|
||||
@@ -1790,24 +1707,6 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
};
|
||||
let observed_usage_candidate = match observed_usage_candidate_result {
|
||||
Ok(candidate) => Some(Bytes::from(candidate)),
|
||||
Err(EcstoreError::ConfigNotFound) => None,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
state = "observed_candidate_load_failed",
|
||||
error = %err,
|
||||
"Scanner skipped an unavailable observed usage candidate for scoped refresh"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
@@ -1822,9 +1721,7 @@ where
|
||||
scan_mode,
|
||||
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: usage_persist_baseline.data.clone(),
|
||||
observed_usage_candidate,
|
||||
requires_full_scan: scheduling.requires_full_scan,
|
||||
service_cohort: scheduling.service_cohort,
|
||||
requires_full_scan,
|
||||
#[cfg(test)]
|
||||
resolved_scope_observer: None,
|
||||
},
|
||||
@@ -1949,10 +1846,10 @@ where
|
||||
.as_ref()
|
||||
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
|
||||
let remote_lease_release_safe = Arc::new(AtomicBool::new(true));
|
||||
let mut usage_publication_result = match publication_defer_reason {
|
||||
let mut usage_persist_outcome = match publication_defer_reason {
|
||||
Some(reason) => {
|
||||
drop(receiver);
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Deferred(reason))
|
||||
DataUsagePersistOutcome::Deferred(reason)
|
||||
}
|
||||
None => {
|
||||
// ScannerIO emits its complete or observational update only after
|
||||
@@ -1967,11 +1864,6 @@ where
|
||||
.as_ref()
|
||||
.map(|(_, grants)| grants.iter().map(|grant| grant.lease.token).collect())
|
||||
.unwrap_or_default();
|
||||
let ack_expectation = scan_result
|
||||
.as_ref()
|
||||
.ok()
|
||||
.filter(|result| result.has_dirty_usage_to_acknowledge())
|
||||
.and_then(ScannerCycleResult::publication_expectation);
|
||||
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
ctx_clone,
|
||||
@@ -1984,7 +1876,6 @@ where
|
||||
remote_lease_deadline,
|
||||
remote_lease_fence,
|
||||
)
|
||||
.with_ack_expectation(ack_expectation)
|
||||
.with_remote_lease_tokens(remote_lease_tokens)
|
||||
.with_lease_release_flag(remote_lease_release_safe_for_task),
|
||||
move || {
|
||||
@@ -2018,7 +1909,7 @@ where
|
||||
error = %err,
|
||||
"Scanner data usage persistence task failed"
|
||||
);
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
DataUsagePersistTaskResult::Cancelled => {
|
||||
debug!(
|
||||
@@ -2030,7 +1921,7 @@ where
|
||||
state = "usage_persist_task_cancelled",
|
||||
"Scanner data usage persistence task cancelled"
|
||||
);
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
DataUsagePersistTaskResult::TimedOut => {
|
||||
error!(
|
||||
@@ -2043,12 +1934,11 @@ where
|
||||
state = "usage_persist_task_timed_out",
|
||||
"Scanner data usage persistence task timed out"
|
||||
);
|
||||
DataUsagePublicationResult::from(DataUsagePersistOutcome::Failed)
|
||||
DataUsagePersistOutcome::Failed
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut usage_persist_outcome = usage_publication_result.outcome();
|
||||
let lease_expired = remote_publication_leases
|
||||
.as_ref()
|
||||
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
|
||||
@@ -2248,21 +2138,20 @@ where
|
||||
};
|
||||
}
|
||||
|
||||
usage_publication_result.restrict_outcome(usage_persist_outcome);
|
||||
let (completion_outcome, scanner_pending_maintenance_work, remote_dirty_usage_acknowledgements) =
|
||||
finalize_scanner_cycle_result(scan_cycle_result, usage_publication_result);
|
||||
finalize_scanner_cycle_result(scan_cycle_result, usage_persist_outcome);
|
||||
let remote_dirty_usage_pending = if remote_dirty_usage_acknowledgements.is_empty() {
|
||||
false
|
||||
} else if let Some(notification_system) = storeapi.scanner_notification_system() {
|
||||
let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
|
||||
let acknowledgement_proof = remote_dirty_usage_acknowledgements.clone();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements.into_iter().map(Into::into).collect();
|
||||
let acknowledgements = remote_dirty_usage_acknowledgements
|
||||
.into_iter()
|
||||
.map(|acknowledgement| (acknowledgement.host, acknowledgement.instance_id, acknowledgement.generation))
|
||||
.collect();
|
||||
remote_dirty_usage_acknowledgement_pending(
|
||||
cycle_info.current,
|
||||
acknowledgement_count,
|
||||
&acknowledgement_proof,
|
||||
notification_system.acknowledge_scanner_dirty_usage(acknowledgements),
|
||||
|| probe_scanner_activity(storeapi.as_ref(), true),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
@@ -2694,7 +2583,6 @@ where
|
||||
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
|
||||
let mut superseded_backoff = ScannerRetryBackoff::default();
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
||||
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
||||
@@ -2904,10 +2792,7 @@ where
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: Some(service_cohort.clone()),
|
||||
},
|
||||
true,
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
@@ -3198,14 +3083,11 @@ where
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: maintenance_features.requires_full_scan(
|
||||
maintenance_generation_seen,
|
||||
scanner_maintenance_generation(),
|
||||
wake_reason,
|
||||
),
|
||||
service_cohort: Some(service_cohort.clone()),
|
||||
},
|
||||
maintenance_features.requires_full_scan(
|
||||
maintenance_generation_seen,
|
||||
scanner_maintenance_generation(),
|
||||
wake_reason,
|
||||
),
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
@@ -3484,35 +3366,21 @@ fn scanner_cycle_completion_outcome(
|
||||
|
||||
fn finalize_scanner_cycle_result(
|
||||
scan_cycle_result: crate::scanner_io::ScannerCycleResult,
|
||||
publication: DataUsagePublicationResult,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
) -> (ScannerCycleOutcome, bool, Vec<ScannerDirtyUsageAcknowledgement>) {
|
||||
let (usage_persist_outcome, proof) = publication.into_parts();
|
||||
let completion_outcome = scanner_cycle_completion_outcome_for_result(&scan_cycle_result, usage_persist_outcome);
|
||||
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work();
|
||||
let durable_complete_snapshot = scan_cycle_result.status == ScannerCycleStatus::Complete
|
||||
&& matches!(
|
||||
usage_persist_outcome,
|
||||
DataUsagePersistOutcome::Saved | DataUsagePersistOutcome::AlreadyDurable
|
||||
)
|
||||
&& scan_cycle_result.publication_expectation().as_ref().is_some_and(|expected| {
|
||||
proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| proof.verified_version_for(expected).is_some())
|
||||
});
|
||||
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work()
|
||||
|| (scan_cycle_result.has_dirty_usage_to_acknowledge() && !durable_complete_snapshot);
|
||||
);
|
||||
let remote_dirty_usage_acknowledgements = if durable_complete_snapshot {
|
||||
match proof {
|
||||
Some(proof) => scan_cycle_result.acknowledge_durable_usage(&proof),
|
||||
None => Vec::new(),
|
||||
}
|
||||
scan_cycle_result.acknowledge_durable_usage()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
(
|
||||
completion_outcome,
|
||||
pending_maintenance_work || crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
remote_dirty_usage_acknowledgements,
|
||||
)
|
||||
(completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements)
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome_for_result(
|
||||
@@ -3612,17 +3480,13 @@ use activity::*;
|
||||
use backlog::*;
|
||||
use cycle_state::*;
|
||||
use leadership::*;
|
||||
pub(crate) use usage_store::RootPublicationProof;
|
||||
use usage_store::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use activity::scanner_node_activity_for_tests;
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity,
|
||||
scanner_activity_allows_usage_publication, scanner_activity_dirty_usage_state_for_host,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
|
||||
scanner_dirty_usage_acknowledgements,
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
||||
scanner_activity_structural_digest, scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
@@ -3632,11 +3496,8 @@ pub use backlog::{
|
||||
#[cfg(test)]
|
||||
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
|
||||
pub use cycle_state::{
|
||||
SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD, ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus,
|
||||
ScannerRecoveryIntentAcceptResult, ScannerRecoveryIntentConflict, ScannerRecoveryIntentRecord, ScannerRecoveryIntentRequest,
|
||||
ScannerUsageStateResetResult, accept_scanner_usage_recovery_intent, get_scanner_usage_recovery_intent,
|
||||
reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild, run_scanner_usage_recovery_intent,
|
||||
scanner_cycle_recovery_status, scanner_recovery_actor_sha256,
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerUsageStateResetResult, reset_scanner_cycle_recovery,
|
||||
reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
|
||||
};
|
||||
pub(crate) use cycle_state::{
|
||||
current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use super::*;
|
||||
use crate::storage_api::ScannerStorage;
|
||||
use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum ScannerCycleWakeReason {
|
||||
@@ -52,25 +51,18 @@ pub(crate) fn scanner_cycle_outcome_with_pending_maintenance(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn remote_dirty_usage_acknowledgement_pending<F, E, C, CF>(
|
||||
pub(super) async fn remote_dirty_usage_acknowledgement_pending<F, E>(
|
||||
cycle: u64,
|
||||
acknowledgement_count: usize,
|
||||
acknowledgements: &[ScannerDirtyUsageAcknowledgement],
|
||||
acknowledgement: F,
|
||||
confirm_after_error: C,
|
||||
) -> bool
|
||||
where
|
||||
F: Future<Output = Result<bool, E>>,
|
||||
E: std::fmt::Display,
|
||||
C: FnOnce() -> CF,
|
||||
CF: Future<Output = Result<ScannerActivitySnapshot, String>>,
|
||||
{
|
||||
match acknowledgement.await {
|
||||
Ok(dirty_usage_pending) => dirty_usage_pending,
|
||||
Err(err) => {
|
||||
if remote_dirty_usage_acknowledgement_loss_reconciled(acknowledgements, confirm_after_error().await) {
|
||||
return false;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
@@ -87,29 +79,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn remote_dirty_usage_acknowledgement_loss_reconciled(
|
||||
acknowledgements: &[ScannerDirtyUsageAcknowledgement],
|
||||
activity_after_error: Result<ScannerActivitySnapshot, String>,
|
||||
) -> bool {
|
||||
if acknowledgements.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(activity_after_error) = activity_after_error else {
|
||||
return false;
|
||||
};
|
||||
if !scanner_activity_allows_usage_publication(&activity_after_error) {
|
||||
return false;
|
||||
}
|
||||
let mut acknowledged_hosts = HashSet::with_capacity(acknowledgements.len());
|
||||
acknowledgements.iter().all(|acknowledgement| {
|
||||
if !acknowledged_hosts.insert(acknowledgement.host.as_str()) {
|
||||
return false;
|
||||
}
|
||||
scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host)
|
||||
.is_some_and(|(instance_id, _generation, pending)| instance_id == acknowledgement.instance_id && !pending)
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) struct ScannerCleanIdleBackoff {
|
||||
pub(super) interval_multiplier: u32,
|
||||
@@ -472,59 +441,11 @@ pub(crate) struct ScannerNodeActivity {
|
||||
|
||||
pub(crate) type ScannerActivitySnapshot = BTreeMap<String, ScannerNodeActivity>;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn scanner_node_activity_for_tests(
|
||||
instance_id: &str,
|
||||
namespace_generation: u64,
|
||||
dirty_usage_generation: u64,
|
||||
dirty_usage_pending: bool,
|
||||
) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation: 0,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
topology_digest: [0; 32],
|
||||
data_movement_active: false,
|
||||
dirty_usage_generation,
|
||||
dirty_usage_pending,
|
||||
movement_generation: 0,
|
||||
publication_blocked: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerDirtyUsageAcknowledgement {
|
||||
pub(crate) host: String,
|
||||
pub(crate) instance_id: String,
|
||||
pub(crate) kind: ScannerDirtyUsageAcknowledgementKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerDirtyUsageAcknowledgementKind {
|
||||
Generation(u64),
|
||||
Scoped {
|
||||
owner_id: String,
|
||||
entries: Vec<crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<ScannerDirtyUsageAcknowledgement> for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement {
|
||||
fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self {
|
||||
match acknowledgement.kind {
|
||||
ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Self::Generation {
|
||||
host: acknowledgement.host,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
generation,
|
||||
},
|
||||
ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } => Self::Scoped {
|
||||
host: acknowledgement.host,
|
||||
owner_id,
|
||||
instance_id: acknowledgement.instance_id,
|
||||
entries,
|
||||
},
|
||||
}
|
||||
}
|
||||
pub(crate) generation: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -1053,7 +974,7 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
|
||||
.map(|(host, activity)| ScannerDirtyUsageAcknowledgement {
|
||||
host: host.clone(),
|
||||
instance_id: activity.instance_id.clone(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(activity.dirty_usage_generation),
|
||||
generation: activity.dirty_usage_generation,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -18,9 +18,6 @@ use crate::data_usage_define::{
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup,
|
||||
};
|
||||
use crate::storage_api::owner::ObjectIO as _;
|
||||
use rustfs_filemeta::MetaCacheEntry;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
const SCANNER_CYCLE_RECOVERY_SCHEMA_VERSION: u16 = 1;
|
||||
@@ -36,97 +33,6 @@ const LEGACY_INCOMPLETE_USAGE_FLOOR_RECOVERY: &str = "legacy_empty_usage_floor";
|
||||
const CACHE_CYCLE_AHEAD: &str = "cache_cycle_ahead";
|
||||
|
||||
const SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD: &str = "full-rebuild";
|
||||
const SCANNER_RECOVERY_INTENT_SCHEMA_VERSION: u16 = 1;
|
||||
const SCANNER_RECOVERY_INTENT_PREFIX: &str = ".usage.v2.recovery-intents";
|
||||
const MAX_SCANNER_RECOVERY_INTENT_BYTES: u64 = 16 * 1024;
|
||||
const SCANNER_RECOVERY_INTENT_STATE_ACCEPTED: &str = "accepted";
|
||||
const SCANNER_RECOVERY_INTENT_STATE_RUNNING: &str = "running";
|
||||
const SCANNER_RECOVERY_INTENT_STATE_COMPLETED: &str = "completed";
|
||||
const SCANNER_RECOVERY_INTENT_STATE_FAILED: &str = "failed";
|
||||
pub const SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD: &str = "scanner-usage-full-rebuild";
|
||||
const MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES: usize = 4096;
|
||||
const SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE: usize = 128;
|
||||
const MAX_SCANNER_RECOVERY_INTENT_STARTUP_REPLAY: usize = 64;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) mod cleanup_io_fault {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(in crate::scanner) enum Stage {
|
||||
PrimaryRead,
|
||||
PrimaryWrite,
|
||||
UsageFence,
|
||||
}
|
||||
|
||||
struct Injection {
|
||||
store: std::sync::Weak<ECStore>,
|
||||
stage: Stage,
|
||||
fired: AtomicBool,
|
||||
owned: AtomicBool,
|
||||
newer_completion: bool,
|
||||
}
|
||||
|
||||
static INJECTION: StdMutex<Option<Arc<Injection>>> = StdMutex::new(None);
|
||||
pub(in crate::scanner) struct Guard(Arc<Injection>);
|
||||
|
||||
impl Guard {
|
||||
pub(in crate::scanner) fn fired_while_owned(&self) -> bool {
|
||||
self.0.fired.load(Ordering::Relaxed) && self.0.owned.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = INJECTION.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.0)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner) fn install(store: &Arc<ECStore>, stage: Stage, newer_completion: bool) -> Guard {
|
||||
let injection = Arc::new(Injection {
|
||||
store: Arc::downgrade(store),
|
||||
stage,
|
||||
fired: AtomicBool::new(false),
|
||||
owned: AtomicBool::new(false),
|
||||
newer_completion,
|
||||
});
|
||||
let mut slot = INJECTION.lock().expect("cleanup injection slot");
|
||||
assert!(slot.is_none(), "only one cleanup I/O injection may be installed");
|
||||
*slot = Some(injection.clone());
|
||||
Guard(injection)
|
||||
}
|
||||
|
||||
pub(super) fn check(store: &Arc<ECStore>, stage: Stage, owned: bool) -> Result<(), ScannerError> {
|
||||
let injection = {
|
||||
let mut slot = INJECTION.lock().expect("cleanup injection slot");
|
||||
if slot
|
||||
.as_ref()
|
||||
.is_some_and(|injection| injection.stage == stage && injection.store.ptr_eq(&Arc::downgrade(store)))
|
||||
{
|
||||
slot.take()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let Some(injection) = injection else {
|
||||
return Ok(());
|
||||
};
|
||||
injection.fired.store(true, Ordering::Relaxed);
|
||||
injection.owned.store(owned, Ordering::Relaxed);
|
||||
if injection.newer_completion {
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
}
|
||||
let reason = match stage {
|
||||
Stage::PrimaryRead => "injected primary read failure",
|
||||
Stage::PrimaryWrite => "injected primary write failure",
|
||||
Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
Err(ScannerError::Io(std::io::Error::other(reason)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct ScannerCycleRecoveryStatus {
|
||||
@@ -159,43 +65,6 @@ pub struct ScannerUsageStateResetResult {
|
||||
pub reset_paths: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ScannerRecoveryIntentRequest {
|
||||
pub action: String,
|
||||
pub mode: String,
|
||||
pub idempotency_key: String,
|
||||
pub actor_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ScannerRecoveryIntentRecord {
|
||||
pub schema_version: u16,
|
||||
pub intent_id: String,
|
||||
pub action: String,
|
||||
pub mode: String,
|
||||
pub state: String,
|
||||
pub actor_sha256: String,
|
||||
pub idempotency_key_sha256: String,
|
||||
pub request_sha256: String,
|
||||
pub accepted_at_unix_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||
pub struct ScannerRecoveryIntentConflict {
|
||||
pub intent_id: String,
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub enum ScannerRecoveryIntentAcceptResult {
|
||||
Accepted { record: ScannerRecoveryIntentRecord },
|
||||
Replayed { record: ScannerRecoveryIntentRecord },
|
||||
Conflict { existing: ScannerRecoveryIntentConflict },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct ScannerUsageStateResetSlot {
|
||||
path: String,
|
||||
@@ -212,8 +81,6 @@ static SCANNER_CYCLE_RECOVERY_STATUS: LazyLock<RwLock<ScannerCycleRecoveryStatus
|
||||
..Default::default()
|
||||
})
|
||||
});
|
||||
// An old startup observation must not overwrite a newer explicit reset status.
|
||||
static SCANNER_CYCLE_RECOVERY_STATUS_VERSION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus {
|
||||
SCANNER_CYCLE_RECOVERY_STATUS
|
||||
@@ -223,24 +90,6 @@ pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus {
|
||||
}
|
||||
|
||||
fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) {
|
||||
let _ = publish_scanner_cleanup_status(status, None);
|
||||
}
|
||||
|
||||
pub(super) fn scanner_cleanup_status_version() -> u64 {
|
||||
let _status = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(super) fn publish_scanner_cleanup_status(status: ScannerCycleRecoveryStatus, expected: Option<u64>) -> Option<u64> {
|
||||
let mut current = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let version = SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed);
|
||||
if expected.is_some_and(|expected| expected == u64::MAX || expected != version) {
|
||||
return None;
|
||||
}
|
||||
let recovery_required = if matches!(
|
||||
status.state.as_str(),
|
||||
"blocked"
|
||||
@@ -257,27 +106,9 @@ pub(super) fn publish_scanner_cleanup_status(status: ScannerCycleRecoveryStatus,
|
||||
};
|
||||
metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED).set(recovery_required);
|
||||
metrics::gauge!(METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT).set(status.retry_count as f64);
|
||||
*current = status;
|
||||
let next = version.saturating_add(1);
|
||||
SCANNER_CYCLE_RECOVERY_STATUS_VERSION.store(next, Ordering::Relaxed);
|
||||
Some(next)
|
||||
}
|
||||
|
||||
fn publish_scanner_cleanup_failure(reason: String, expected: u64) {
|
||||
let mut status = {
|
||||
let current = SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if SCANNER_CYCLE_RECOVERY_STATUS_VERSION.load(Ordering::Relaxed) != expected {
|
||||
return;
|
||||
}
|
||||
current.clone()
|
||||
};
|
||||
// Preserve the latest core progress, including a newly written primary's
|
||||
// revision and epoch. The original marker may predate that durable write.
|
||||
status.reason = Some(reason);
|
||||
status.last_attempt_at_unix_secs = Some(unix_now_secs());
|
||||
let _ = publish_scanner_cleanup_status(status, Some(expected));
|
||||
*SCANNER_CYCLE_RECOVERY_STATUS
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner()) = status;
|
||||
}
|
||||
|
||||
pub(super) fn record_scanner_usage_floor_failure(reason: String) {
|
||||
@@ -410,445 +241,6 @@ fn unix_now_secs() -> u64 {
|
||||
u64::try_from(Utc::now().timestamp()).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn sha256_hex(parts: &[&[u8]]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for part in parts {
|
||||
hasher.update(part);
|
||||
hasher.update([0]);
|
||||
}
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let digest = hasher.finalize();
|
||||
let mut encoded = String::with_capacity(64);
|
||||
for byte in digest {
|
||||
encoded.push(char::from(HEX[usize::from(byte >> 4)]));
|
||||
encoded.push(char::from(HEX[usize::from(byte & 0x0f)]));
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
fn is_canonical_sha256(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn validate_idempotency_key(key: &str) -> Result<(), ScannerError> {
|
||||
if !(8..=256).contains(&key.len()) {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner recovery intent idempotency key length is unsupported".to_string(),
|
||||
));
|
||||
}
|
||||
if !key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii() && !byte.is_ascii_control() && !byte.is_ascii_whitespace())
|
||||
{
|
||||
return Err(ScannerError::Other(
|
||||
"scanner recovery intent idempotency key must be printable ASCII without whitespace".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_recovery_intent_request(request: &ScannerRecoveryIntentRequest) -> Result<(), ScannerError> {
|
||||
if request.action != SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD {
|
||||
return Err(ScannerError::Other("scanner recovery intent action is unsupported".to_string()));
|
||||
}
|
||||
if request.mode != SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD {
|
||||
return Err(ScannerError::Other("scanner recovery intent mode is unsupported".to_string()));
|
||||
}
|
||||
if !is_canonical_sha256(&request.actor_sha256) {
|
||||
return Err(ScannerError::Other("scanner recovery intent actor identity is invalid".to_string()));
|
||||
}
|
||||
validate_idempotency_key(&request.idempotency_key)
|
||||
}
|
||||
|
||||
fn scanner_recovery_intent_path(intent_id: &str) -> Result<String, ScannerError> {
|
||||
if !is_canonical_sha256(intent_id) {
|
||||
return Err(ScannerError::Other("scanner recovery intent id is invalid".to_string()));
|
||||
}
|
||||
Ok(format!("{SCANNER_RECOVERY_INTENT_PREFIX}/{intent_id}.json"))
|
||||
}
|
||||
|
||||
fn scanner_recovery_intent_id_from_entry_name(entry_name: &str) -> Option<String> {
|
||||
let relative = entry_name
|
||||
.strip_prefix(SCANNER_RECOVERY_INTENT_PREFIX)
|
||||
.and_then(|name| name.strip_prefix('/'))?;
|
||||
let intent_id = relative.strip_suffix(".json")?;
|
||||
(!intent_id.contains('/') && is_canonical_sha256(intent_id)).then(|| intent_id.to_string())
|
||||
}
|
||||
|
||||
pub fn scanner_recovery_actor_sha256(actor: &str) -> String {
|
||||
sha256_hex(&[b"scanner-recovery-actor-v1", actor.as_bytes()])
|
||||
}
|
||||
|
||||
fn scanner_recovery_intent_candidate(
|
||||
request: &ScannerRecoveryIntentRequest,
|
||||
) -> Result<ScannerRecoveryIntentRecord, ScannerError> {
|
||||
validate_recovery_intent_request(request)?;
|
||||
let intent_id = sha256_hex(&[
|
||||
b"scanner-recovery-intent-v1",
|
||||
request.actor_sha256.as_bytes(),
|
||||
request.idempotency_key.as_bytes(),
|
||||
]);
|
||||
Ok(ScannerRecoveryIntentRecord {
|
||||
schema_version: SCANNER_RECOVERY_INTENT_SCHEMA_VERSION,
|
||||
intent_id,
|
||||
action: request.action.clone(),
|
||||
mode: request.mode.clone(),
|
||||
state: SCANNER_RECOVERY_INTENT_STATE_ACCEPTED.to_string(),
|
||||
actor_sha256: request.actor_sha256.clone(),
|
||||
idempotency_key_sha256: sha256_hex(&[b"scanner-recovery-idempotency-key-v1", request.idempotency_key.as_bytes()]),
|
||||
request_sha256: sha256_hex(&[
|
||||
b"scanner-recovery-request-v1",
|
||||
request.actor_sha256.as_bytes(),
|
||||
request.action.as_bytes(),
|
||||
request.mode.as_bytes(),
|
||||
request.idempotency_key.as_bytes(),
|
||||
]),
|
||||
accepted_at_unix_secs: unix_now_secs(),
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_recovery_intent_record(record: &ScannerRecoveryIntentRecord) -> Result<(), ScannerError> {
|
||||
if record.schema_version != SCANNER_RECOVERY_INTENT_SCHEMA_VERSION {
|
||||
return Err(ScannerError::Other("scanner recovery intent schema is unsupported".to_string()));
|
||||
}
|
||||
if !is_canonical_sha256(&record.intent_id)
|
||||
|| !is_canonical_sha256(&record.actor_sha256)
|
||||
|| !is_canonical_sha256(&record.idempotency_key_sha256)
|
||||
|| !is_canonical_sha256(&record.request_sha256)
|
||||
{
|
||||
return Err(ScannerError::Other("scanner recovery intent identity is invalid".to_string()));
|
||||
}
|
||||
if record.action != SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD
|
||||
|| record.mode != SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD
|
||||
|| !matches!(
|
||||
record.state.as_str(),
|
||||
SCANNER_RECOVERY_INTENT_STATE_ACCEPTED
|
||||
| SCANNER_RECOVERY_INTENT_STATE_RUNNING
|
||||
| SCANNER_RECOVERY_INTENT_STATE_COMPLETED
|
||||
| SCANNER_RECOVERY_INTENT_STATE_FAILED
|
||||
)
|
||||
{
|
||||
return Err(ScannerError::Other("scanner recovery intent state is invalid".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_recovery_intent_record(data: &[u8]) -> Result<ScannerRecoveryIntentRecord, ScannerError> {
|
||||
let record: ScannerRecoveryIntentRecord =
|
||||
serde_json::from_slice(data).map_err(|err| ScannerError::Other(format!("scanner recovery intent is invalid: {err}")))?;
|
||||
validate_recovery_intent_record(&record)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
fn compare_recovery_intent(
|
||||
expected: &ScannerRecoveryIntentRecord,
|
||||
existing: ScannerRecoveryIntentRecord,
|
||||
) -> ScannerRecoveryIntentAcceptResult {
|
||||
if existing.actor_sha256 == expected.actor_sha256
|
||||
&& existing.action == expected.action
|
||||
&& existing.mode == expected.mode
|
||||
&& existing.idempotency_key_sha256 == expected.idempotency_key_sha256
|
||||
&& existing.request_sha256 == expected.request_sha256
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Replayed { record: existing }
|
||||
} else {
|
||||
ScannerRecoveryIntentAcceptResult::Conflict {
|
||||
existing: ScannerRecoveryIntentConflict {
|
||||
intent_id: existing.intent_id,
|
||||
state: existing.state,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_recovery_intent_record(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
path: &str,
|
||||
) -> Result<Option<ScannerRecoveryIntentRecord>, ScannerError> {
|
||||
read_recovery_intent_record_with_revision(storeapi, path)
|
||||
.await
|
||||
.map(|record| record.map(|(record, _)| record))
|
||||
}
|
||||
|
||||
async fn read_recovery_intent_record_with_revision(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
path: &str,
|
||||
) -> Result<Option<(ScannerRecoveryIntentRecord, DataUsageCacheRevision)>, ScannerError> {
|
||||
let mut reader = match storeapi
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ScannerObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(
|
||||
EcstoreError::FileNotFound
|
||||
| EcstoreError::VolumeNotFound
|
||||
| EcstoreError::ObjectNotFound(_, _)
|
||||
| EcstoreError::BucketNotFound(_)
|
||||
| EcstoreError::ConfigNotFound,
|
||||
) => return Ok(None),
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to read scanner recovery intent: {err}"))),
|
||||
};
|
||||
let revision = reader
|
||||
.object_info
|
||||
.etag
|
||||
.as_ref()
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| ScannerError::Other("scanner recovery intent has no revision".to_string()))?;
|
||||
let max_object_size = i64::try_from(MAX_SCANNER_RECOVERY_INTENT_BYTES).unwrap_or(i64::MAX);
|
||||
if reader.object_info.is_dir || reader.object_info.size < 0 || reader.object_info.size > max_object_size {
|
||||
return Err(ScannerError::Other("scanner recovery intent exceeds the bounded object size".to_string()));
|
||||
}
|
||||
let max_len = usize::try_from(MAX_SCANNER_RECOVERY_INTENT_BYTES).unwrap_or(usize::MAX);
|
||||
let mut data = Vec::new();
|
||||
(&mut reader)
|
||||
.take(MAX_SCANNER_RECOVERY_INTENT_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut data)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to read scanner recovery intent: {err}")))?;
|
||||
if data.is_empty() {
|
||||
return Err(ScannerError::Other("scanner recovery intent is empty".to_string()));
|
||||
}
|
||||
if data.len() > max_len {
|
||||
return Err(ScannerError::Other("scanner recovery intent exceeds the bounded object size".to_string()));
|
||||
}
|
||||
decode_recovery_intent_record(&data).map(|record| Some((record, revision)))
|
||||
}
|
||||
|
||||
pub async fn get_scanner_usage_recovery_intent(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
intent_id: &str,
|
||||
) -> Result<Option<ScannerRecoveryIntentRecord>, ScannerError> {
|
||||
let path = scanner_recovery_intent_path(intent_id)?;
|
||||
read_recovery_intent_record(storeapi, &path).await
|
||||
}
|
||||
|
||||
fn scanner_recovery_intent_is_replayable(record: &ScannerRecoveryIntentRecord) -> bool {
|
||||
matches!(
|
||||
record.state.as_str(),
|
||||
SCANNER_RECOVERY_INTENT_STATE_ACCEPTED | SCANNER_RECOVERY_INTENT_STATE_RUNNING
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn scanner_usage_recovery_intents_for_startup(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
) -> Result<Vec<String>, ScannerError> {
|
||||
let discovered = Arc::new(StdMutex::new(BTreeSet::<String>::new()));
|
||||
for set in storeapi.all_set_disks() {
|
||||
if ctx.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
let disks = set.get_local_disks().await;
|
||||
if disks.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let read_quorum = disks.len().saturating_sub(set.default_parity_count).clamp(1, disks.len());
|
||||
let mut forward_to = None;
|
||||
loop {
|
||||
let page_entries = Arc::new(StdMutex::new(Vec::<String>::new()));
|
||||
let page_entries_for_set = page_entries.clone();
|
||||
let list_result = list_path_raw(
|
||||
ctx.child_token(),
|
||||
ListPathRawOptions {
|
||||
disks: disks.clone(),
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
path: SCANNER_RECOVERY_INTENT_PREFIX.to_string(),
|
||||
recursive: true,
|
||||
skip_hidden_prefix_check: true,
|
||||
forward_to: forward_to.clone(),
|
||||
min_disks: read_quorum,
|
||||
report_not_found: true,
|
||||
per_disk_limit: i32::try_from(SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE).unwrap_or(i32::MAX),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
let page_entries = page_entries_for_set.clone();
|
||||
Box::pin(async move {
|
||||
if scanner_recovery_intent_id_from_entry_name(&entry.name).is_some() {
|
||||
page_entries
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.push(entry.name);
|
||||
}
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match list_result {
|
||||
Ok(()) => {}
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => break,
|
||||
Err(err) => {
|
||||
return Err(ScannerError::Other(format!("failed to list scanner recovery intents for startup: {err}")));
|
||||
}
|
||||
}
|
||||
|
||||
let page = {
|
||||
let mut guard = page_entries.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
guard.sort();
|
||||
guard.dedup();
|
||||
std::mem::take(&mut *guard)
|
||||
};
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
{
|
||||
let mut all = discovered.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
for entry_name in &page {
|
||||
if let Some(intent_id) = scanner_recovery_intent_id_from_entry_name(entry_name) {
|
||||
all.insert(intent_id);
|
||||
}
|
||||
if all.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if all.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if page.len() < SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE {
|
||||
break;
|
||||
}
|
||||
forward_to = page.last().cloned();
|
||||
}
|
||||
}
|
||||
|
||||
let intent_ids = {
|
||||
let guard = discovered.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
guard.iter().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
let mut replayable = Vec::new();
|
||||
for intent_id in intent_ids {
|
||||
let Some(record) = get_scanner_usage_recovery_intent(storeapi.clone(), &intent_id).await? else {
|
||||
continue;
|
||||
};
|
||||
if scanner_recovery_intent_is_replayable(&record) {
|
||||
replayable.push(record.intent_id);
|
||||
if replayable.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_REPLAY {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(replayable)
|
||||
}
|
||||
|
||||
pub async fn accept_scanner_usage_recovery_intent(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
request: ScannerRecoveryIntentRequest,
|
||||
) -> Result<ScannerRecoveryIntentAcceptResult, ScannerError> {
|
||||
let candidate = scanner_recovery_intent_candidate(&request)?;
|
||||
let path = scanner_recovery_intent_path(&candidate.intent_id)?;
|
||||
let encoded = serde_json::to_vec(&candidate)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode scanner recovery intent: {err}")))?;
|
||||
match save_config_with_preconditions(storeapi.clone(), &path, encoded, DataUsageCacheRevision::Missing.preconditions()).await
|
||||
{
|
||||
Ok(_) => Ok(ScannerRecoveryIntentAcceptResult::Accepted { record: candidate }),
|
||||
Err(EcstoreError::PreconditionFailed) => {
|
||||
let existing = read_recovery_intent_record(storeapi, &path).await?;
|
||||
let Some(existing) = existing else {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner recovery intent disappeared after creation conflict".to_string(),
|
||||
));
|
||||
};
|
||||
Ok(compare_recovery_intent(&candidate, existing))
|
||||
}
|
||||
Err(err) => Err(ScannerError::Other(format!("failed to persist scanner recovery intent: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn transition_scanner_usage_recovery_intent(
|
||||
storeapi: Arc<ECStore>,
|
||||
intent_id: &str,
|
||||
expected_states: &[&str],
|
||||
next_state: &str,
|
||||
) -> Result<Option<ScannerRecoveryIntentRecord>, ScannerError> {
|
||||
let path = scanner_recovery_intent_path(intent_id)?;
|
||||
for _ in 0..8 {
|
||||
let Some((mut record, revision)) = read_recovery_intent_record_with_revision(storeapi.clone(), &path).await? else {
|
||||
return Err(ScannerError::Other("scanner recovery intent disappeared before execution".to_string()));
|
||||
};
|
||||
if record.state == next_state {
|
||||
return Ok(Some(record));
|
||||
}
|
||||
if !expected_states.contains(&record.state.as_str()) {
|
||||
return Ok(None);
|
||||
}
|
||||
record.state = next_state.to_string();
|
||||
let encoded = serde_json::to_vec(&record)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode scanner recovery intent: {err}")))?;
|
||||
match save_config_with_preconditions(storeapi.clone(), &path, encoded, revision.preconditions()).await {
|
||||
Ok(_) => return Ok(Some(record)),
|
||||
Err(EcstoreError::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to persist scanner recovery intent: {err}"))),
|
||||
}
|
||||
}
|
||||
Err(ScannerError::Other(
|
||||
"scanner recovery intent state changed repeatedly during transition".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn run_scanner_usage_recovery_intent(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
intent_id: String,
|
||||
) -> Result<Option<ScannerUsageStateResetResult>, ScannerError> {
|
||||
let Some(record) = transition_scanner_usage_recovery_intent(
|
||||
storeapi.clone(),
|
||||
&intent_id,
|
||||
&[SCANNER_RECOVERY_INTENT_STATE_ACCEPTED, SCANNER_RECOVERY_INTENT_STATE_RUNNING],
|
||||
SCANNER_RECOVERY_INTENT_STATE_RUNNING,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if record.action != SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD
|
||||
|| record.mode != SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD
|
||||
{
|
||||
return Err(ScannerError::Other(
|
||||
"scanner recovery intent action or mode changed before execution".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
match reset_scanner_usage_state_for_full_rebuild(ctx, storeapi.clone()).await {
|
||||
Ok(result) => {
|
||||
transition_scanner_usage_recovery_intent(
|
||||
storeapi,
|
||||
&intent_id,
|
||||
&[SCANNER_RECOVERY_INTENT_STATE_RUNNING, SCANNER_RECOVERY_INTENT_STATE_FAILED],
|
||||
SCANNER_RECOVERY_INTENT_STATE_COMPLETED,
|
||||
)
|
||||
.await?;
|
||||
Ok(Some(result))
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = transition_scanner_usage_recovery_intent(
|
||||
storeapi,
|
||||
&intent_id,
|
||||
&[SCANNER_RECOVERY_INTENT_STATE_RUNNING],
|
||||
SCANNER_RECOVERY_INTENT_STATE_FAILED,
|
||||
)
|
||||
.await;
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn recovery_status(state: &str, reason: Option<&str>, retryable: bool) -> ScannerCycleRecoveryStatus {
|
||||
ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
@@ -1540,81 +932,10 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset a blocked cycle state after an explicit full-rescan request. Durable
|
||||
/// cleanup state fences primary rewrites; marker removal retains its ETag and
|
||||
/// usage-epoch checks.
|
||||
/// Reset a blocked cycle state after an operator has explicitly requested a full
|
||||
/// usage rebuild. The primary object is changed first with its observed ETag;
|
||||
/// the recovery marker is removed only when its own ETag still matches.
|
||||
pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
|
||||
reset_scanner_cycle_recovery_for_intent(ctx, storeapi, None, None).await
|
||||
}
|
||||
|
||||
/// Resume only an operator reset whose cleanup phase is already durable.
|
||||
/// Missing or merely blocked markers never authorize an automatic reset.
|
||||
pub(super) async fn resume_scanner_cycle_cleanup(ctx: CancellationToken, storeapi: Arc<ECStore>) -> Result<(), ScannerError> {
|
||||
let status_version = scanner_cleanup_status_version();
|
||||
let (marker, revision) = match read_scanner_cleanup_marker(storeapi.clone(), &ctx).await {
|
||||
Ok(Some(marker)) => marker,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(error) => {
|
||||
let _ =
|
||||
publish_scanner_cleanup_status(recovery_status("blocked", Some(&error.to_string()), false), Some(status_version));
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let mut observation =
|
||||
publish_scanner_cleanup_status(recovery_status_from_marker(&marker, &marker.state), Some(status_version));
|
||||
if marker.state != "cleanup-pending" {
|
||||
return Ok(());
|
||||
}
|
||||
let result = reset_scanner_cycle_recovery_for_intent(ctx, storeapi, Some(revision), Some(&mut observation)).await;
|
||||
if let Err(error) = &result
|
||||
&& let Some(observation) = observation
|
||||
{
|
||||
publish_scanner_cleanup_failure(error.to_string(), observation);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn read_scanner_cleanup_marker(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
ctx: &CancellationToken,
|
||||
) -> Result<Option<(ScannerCycleRecoveryMarker, DataUsageCacheRevision)>, ScannerError> {
|
||||
let (data, revision) = tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => return Err(ScannerError::Other("scanner cleanup recovery was cancelled".to_string())),
|
||||
result = tokio::time::timeout(data_usage_persist_timeout(), read_cycle_recovery_marker_bytes(storeapi)) => {
|
||||
result.map_err(|_| ScannerError::Other("scanner cleanup marker inspection timed out".to_string()))?
|
||||
.map_err(|err| ScannerError::Other(format!("failed to inspect pending scanner cleanup: {err}")))?
|
||||
}
|
||||
};
|
||||
let Some(data) = data else {
|
||||
return Ok(None);
|
||||
};
|
||||
let marker: ScannerCycleRecoveryMarker = serde_json::from_slice(&data)
|
||||
.map_err(|err| ScannerError::Other(format!("pending scanner cleanup marker is invalid: {err}")))?;
|
||||
validate_recovery_marker(&marker)
|
||||
.map_err(|err| ScannerError::Other(format!("pending scanner cleanup marker is invalid: {err}")))?;
|
||||
Ok(Some((marker, revision)))
|
||||
}
|
||||
|
||||
pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
expected_cleanup_revision: Option<DataUsageCacheRevision>,
|
||||
mut observation: Option<&mut Option<u64>>,
|
||||
) -> Result<(), ScannerError> {
|
||||
#[cfg(test)]
|
||||
let resume_only = expected_cleanup_revision.is_some();
|
||||
// The outer Some distinguishes a tracked resume whose version may be
|
||||
// invalidated from an explicit v3 reset with no observation owner.
|
||||
let mut publish_status = |status| {
|
||||
if let Some(version) = observation.as_deref_mut() {
|
||||
if let Some(expected) = *version {
|
||||
*version = publish_scanner_cleanup_status(status, Some(expected));
|
||||
}
|
||||
} else {
|
||||
set_scanner_cycle_recovery_status(status);
|
||||
}
|
||||
};
|
||||
let lock = storeapi
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
@@ -1643,21 +964,6 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
}
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
|
||||
};
|
||||
if let Some(expected) = expected_cleanup_revision {
|
||||
if marker_revision != expected || !owns_reset() {
|
||||
return Err(ScannerError::Other(
|
||||
"pending scanner cleanup changed before recovery acquired ownership".to_string(),
|
||||
));
|
||||
}
|
||||
let marker: ScannerCycleRecoveryMarker = marker_data
|
||||
.as_deref()
|
||||
.and_then(|data| serde_json::from_slice(data).ok())
|
||||
.filter(|marker| validate_recovery_marker(marker).is_ok() && marker.state == "cleanup-pending")
|
||||
.ok_or_else(|| {
|
||||
ScannerError::Other("scanner cleanup recovery requires an unchanged cleanup-pending marker".to_string())
|
||||
})?;
|
||||
publish_status(recovery_status_from_marker(&marker, "cleanup-pending"));
|
||||
}
|
||||
let Some(marker_data) = marker_data else {
|
||||
// A delete may commit before its reply is lost. Confirm both durable
|
||||
// fences before treating a retry without its marker as completed.
|
||||
@@ -1675,7 +981,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
|
||||
));
|
||||
}
|
||||
publish_status(recovery_status("healthy", None, false));
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
};
|
||||
@@ -1691,10 +997,6 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::PrimaryRead, owns_reset())?;
|
||||
}
|
||||
let (mut primary_reader, primary_revision) = match storeapi
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -1760,7 +1062,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
let (cleanup_marker, cleanup_marker_revision) =
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
|
||||
.await?;
|
||||
publish_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
let fence_epoch = primary_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
@@ -1780,10 +1082,6 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
#[cfg(test)]
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::PrimaryWrite, owns_reset())?;
|
||||
}
|
||||
let preserved_info = save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
@@ -1857,7 +1155,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
|
||||
}
|
||||
})?;
|
||||
publish_status(recovery_status("healthy", None, false));
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
} else if !force_full_rescan && !marker_cleanup_pending {
|
||||
@@ -1930,23 +1228,11 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let usage_fence = fence_scanner_usage_epoch_with_expected_epoch(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
leader_epoch,
|
||||
Some(reset_epoch),
|
||||
false,
|
||||
&owns_reset,
|
||||
);
|
||||
#[cfg(test)]
|
||||
let usage_fence = async {
|
||||
if resume_only {
|
||||
cleanup_io_fault::check(&storeapi, cleanup_io_fault::Stage::UsageFence, owns_reset())?;
|
||||
}
|
||||
usage_fence.await
|
||||
};
|
||||
if let Err(err) = usage_fence.await {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
if let Err(err) =
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
|
||||
.await
|
||||
{
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1974,7 +1260,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
}
|
||||
};
|
||||
if current_revision != rebuilt_revision {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -1995,7 +1281,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
}
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -2032,7 +1318,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
.await
|
||||
{
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -2050,7 +1336,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
"scanner recovery reset deferred by a movement epoch change".to_string(),
|
||||
));
|
||||
}
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "cleanup-pending".to_string(),
|
||||
@@ -2066,7 +1352,7 @@ pub(super) async fn reset_scanner_cycle_recovery_for_intent(
|
||||
});
|
||||
return Err(ScannerError::Other(format!("failed to clear cycle recovery marker: {err}")));
|
||||
}
|
||||
publish_status(ScannerCycleRecoveryStatus {
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
|
||||
state: "healthy".to_string(),
|
||||
|
||||
@@ -27,7 +27,6 @@ use crate::{
|
||||
use serial_test::serial;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::task::Poll;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
@@ -37,11 +36,6 @@ use tokio::time::{Duration, advance};
|
||||
|
||||
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
|
||||
mod quota_reset_preservation;
|
||||
|
||||
mod recovery_control;
|
||||
mod scoped_ack_publication;
|
||||
|
||||
async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
setup_scanner_cycle_store_with_usage_baseline(true).await
|
||||
}
|
||||
@@ -56,17 +50,11 @@ async fn setup_scanner_cycle_store_with_pool_count(
|
||||
) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let store = setup_scanner_cycle_store_at_path(temp_dir.path(), seed_usage_baseline, pool_count).await;
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
async fn setup_scanner_cycle_store_at_path(root: &Path, seed_usage_baseline: bool, pool_count: usize) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = root.join(format!("pool{pool_index}/disk{disk_index}"));
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("scanner cycle test disk should be created");
|
||||
@@ -116,7 +104,7 @@ async fn setup_scanner_cycle_store_at_path(root: &Path, seed_usage_baseline: boo
|
||||
.expect("scanner cycle usage baseline should persist");
|
||||
}
|
||||
|
||||
store
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
async fn restart_scanner_cycle_store_from(store: &Arc<ECStore>) -> Arc<ECStore> {
|
||||
@@ -1231,18 +1219,7 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
1,
|
||||
Arc::clone(&budget),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
),
|
||||
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget), true),
|
||||
)
|
||||
.await
|
||||
.expect("the coordinator must finish its namespace walk while a PUT is pending");
|
||||
@@ -1286,18 +1263,7 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
&mut cycle_info,
|
||||
&mut revision,
|
||||
1,
|
||||
Arc::clone(&retry_budget),
|
||||
ScannerCycleScheduling {
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
},
|
||||
),
|
||||
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget), true),
|
||||
)
|
||||
.await
|
||||
.expect("the same cycle must converge after the pending PUT drains");
|
||||
@@ -5298,103 +5264,6 @@ async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewrit
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_resumes_real_store_cleanup_boundaries_after_reopen() {
|
||||
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let cleanup_paths = [
|
||||
format!("{primary_path}.bkp"),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
|
||||
];
|
||||
|
||||
for completed in 0..=cleanup_paths.len() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 42,
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
encode_scanner_cycle_state(&cycle, 3).expect("cycle state should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("cycle state should persist");
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
|
||||
save_config(
|
||||
store.clone(),
|
||||
primary_path,
|
||||
serde_json::to_vec(&marker).expect("usage reset marker should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("usage reset marker should persist");
|
||||
|
||||
for path in cleanup_paths.iter().skip(completed) {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
usage.scanner_cycle = Some(12);
|
||||
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("cleanup slot should encode"))
|
||||
.await
|
||||
.expect("cleanup slot should persist");
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
save_config(store.clone(), path, b"retain".to_vec())
|
||||
.await
|
||||
.expect("unrelated state should persist before reopen");
|
||||
}
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let intent_before = read_config_with_revision(restarted.clone(), primary_path)
|
||||
.await
|
||||
.expect("reopened reset intent should be readable");
|
||||
|
||||
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), restarted.clone())
|
||||
.await
|
||||
.expect("reopened usage reset should complete");
|
||||
|
||||
assert_eq!(result.leader_epoch, 3, "boundary {completed}");
|
||||
assert_eq!(result.next_cycle, 42, "boundary {completed}");
|
||||
assert_eq!(result.reset_paths.len(), cleanup_paths.len() + 1 - completed, "boundary {completed}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(restarted.clone(), primary_path)
|
||||
.await
|
||||
.expect("completed reset intent should remain readable"),
|
||||
intent_before,
|
||||
"boundary {completed}: resumed cleanup must not rewrite the reset intent"
|
||||
);
|
||||
|
||||
let (floor, state) = persisted_usage_floor_for_startup(restarted.clone(), false)
|
||||
.await
|
||||
.expect("completed reset marker should remain resumable");
|
||||
assert_eq!(floor.leader_epoch, 3, "boundary {completed}");
|
||||
assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending, "boundary {completed}");
|
||||
assert!(
|
||||
persisted_usage_floor(restarted.clone()).await.is_err(),
|
||||
"boundary {completed}: bootstrap marker must not become an authoritative floor"
|
||||
);
|
||||
|
||||
for path in &cleanup_paths {
|
||||
assert!(
|
||||
matches!(read_config(restarted.clone(), path).await, Err(EcstoreError::ConfigNotFound)),
|
||||
"boundary {completed}: reset should remove stale usage slot {path}"
|
||||
);
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
assert_eq!(
|
||||
read_config(restarted.clone(), path)
|
||||
.await
|
||||
.expect("unrelated state should survive reopened reset"),
|
||||
b"retain",
|
||||
"boundary {completed}: reset must preserve non-scanner-state config"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -6291,7 +6160,7 @@ async fn coordinator_classifies_an_expired_publication_lease() {
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
outcome.outcome(),
|
||||
outcome,
|
||||
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
|
||||
);
|
||||
assert!(store.put_counts.lock().await.is_empty(), "expired lease must prevent a PUT");
|
||||
@@ -7432,7 +7301,7 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_a_saved_enum_without_proof_keeps_dirty_pending() {
|
||||
fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
@@ -7440,23 +7309,21 @@ fn finalizing_a_saved_enum_without_proof_keeps_dirty_pending() {
|
||||
let remote_acknowledgement = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
generation: 11,
|
||||
};
|
||||
let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone()))
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate.into());
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Failed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
|
||||
let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot))
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement]);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved.into());
|
||||
.with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(pending);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
assert_eq!(acknowledgements, vec![remote_acknowledgement]);
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -7468,7 +7335,7 @@ fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
|
||||
let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
|
||||
let (outcome, _, acknowledgements) =
|
||||
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement).into());
|
||||
finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7488,7 +7355,7 @@ fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
|
||||
)
|
||||
.with_observational_snapshot_published(true);
|
||||
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved.into());
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved);
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Partial);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7498,28 +7365,13 @@ fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
let acknowledgements = Vec::new();
|
||||
let pending = remote_dirty_usage_acknowledgement_pending(
|
||||
7,
|
||||
1,
|
||||
&acknowledgements,
|
||||
std::future::ready(Ok::<bool, std::io::Error>(true)),
|
||||
|| async { Ok(BTreeMap::new()) },
|
||||
)
|
||||
.await;
|
||||
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
);
|
||||
|
||||
let cleared = remote_dirty_usage_acknowledgement_pending(
|
||||
7,
|
||||
1,
|
||||
&acknowledgements,
|
||||
std::future::ready(Ok::<bool, std::io::Error>(false)),
|
||||
|| async { Ok(BTreeMap::new()) },
|
||||
)
|
||||
.await;
|
||||
let cleared = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(false))).await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared),
|
||||
ScannerCycleOutcome::Completed
|
||||
@@ -7528,9 +7380,7 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
let failed = remote_dirty_usage_acknowledgement_pending(
|
||||
7,
|
||||
1,
|
||||
&acknowledgements,
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("injected acknowledgement failure"))),
|
||||
|| async { Err("confirmation probe failed".to_string()) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
@@ -7539,92 +7389,19 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() {
|
||||
let acknowledgement = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
|
||||
};
|
||||
let cleared_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let response_lost = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("response lost after peer ack"))),
|
||||
|| async { Ok(cleared_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, response_lost),
|
||||
ScannerCycleOutcome::Completed,
|
||||
"a same-instance activity confirmation with no dirty work closes the uncertain ACK"
|
||||
);
|
||||
|
||||
let duplicate_acknowledgements = vec![acknowledgement.clone(), acknowledgement.clone()];
|
||||
let duplicate_target = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
duplicate_acknowledgements.len(),
|
||||
&duplicate_acknowledgements,
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("duplicate target rejected before peer ack"))),
|
||||
|| async { Ok(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, duplicate_target),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a request rejected before peer delivery cannot be recovered by a clean activity snapshot"
|
||||
);
|
||||
|
||||
let restarted_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 7, 3))]);
|
||||
let peer_restarted = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("response lost before restart was observed"))),
|
||||
|| async { Ok(restarted_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, peer_restarted),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a new peer instance cannot confirm whether the old ACK reached durable dirty state"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
let concurrent_write = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
&[acknowledgement],
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other("response lost before concurrent write"))),
|
||||
|| async { Ok(BTreeMap::from([("node-2".to_string(), written_activity)])) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, concurrent_write),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"new dirty usage on the same peer must keep maintenance pending"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_an_already_durable_enum_without_proof_keeps_dirty_pending() {
|
||||
fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() {
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
let (outcome, pending, acknowledgements) =
|
||||
finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable.into());
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable);
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(pending);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
crate::scanner_io::clear_dirty_usage_bucket("photos");
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -7635,8 +7412,7 @@ fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() {
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
|
||||
let (outcome, _, acknowledgements) =
|
||||
finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable.into());
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable);
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -7652,7 +7428,7 @@ fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() {
|
||||
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot));
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved.into());
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved);
|
||||
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Superseded);
|
||||
assert!(acknowledgements.is_empty());
|
||||
@@ -8738,56 +8514,6 @@ async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn service_cohort_aging_preserves_explicit_cycle_wait() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(3600),
|
||||
cycle_interval_source: ScannerRuntimeConfigSource::Env,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = ScannerCycleObservedGenerations::for_wait(
|
||||
&config,
|
||||
None,
|
||||
crate::scanner_io::dirty_usage_generation(),
|
||||
crate::runtime_config::scanner_runtime_config_generation(),
|
||||
crate::scanner_io::scanner_maintenance_generation(),
|
||||
);
|
||||
assert_eq!(observed.dirty_usage, None);
|
||||
let inventory = HashMap::from([(
|
||||
crate::data_usage_define::DataUsageCacheSource::new(0, 0),
|
||||
vec![crate::storage_api::scanner_io::BucketInfo {
|
||||
name: "waiting-bootstrap".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
)]);
|
||||
let mut cohort = crate::scanner_io::ScannerServiceCohort::default();
|
||||
cohort.refresh(&inventory);
|
||||
let ctx = CancellationToken::new();
|
||||
let mut wait = Box::pin(wait_for_next_scanner_cycle(
|
||||
&ctx,
|
||||
config.cycle_interval,
|
||||
observed.dirty_usage,
|
||||
observed.runtime_config,
|
||||
observed.maintenance,
|
||||
|| false,
|
||||
));
|
||||
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
||||
for _ in 0..59 {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
cohort.refresh(&inventory);
|
||||
crate::scanner_io::record_dirty_usage_bucket("hot");
|
||||
assert!(
|
||||
matches!(futures::poll!(&mut wait), Poll::Pending),
|
||||
"aging/dirty must not shorten the explicit hour"
|
||||
);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(wait.await, ScannerCycleWakeReason::Timer);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() {
|
||||
@@ -9076,12 +8802,12 @@ fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acqui
|
||||
ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
|
||||
generation: 5,
|
||||
},
|
||||
]);
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(
|
||||
result,
|
||||
DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")).into(),
|
||||
DataUsagePersistOutcome::Deferred(reason.expect("changed namespace should defer publication")),
|
||||
);
|
||||
assert_eq!(
|
||||
outcome,
|
||||
@@ -9236,7 +8962,7 @@ fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
||||
vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-3".to_string(),
|
||||
instance_id: "epoch-dirty".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Generation(11),
|
||||
generation: 11,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::owner::ObjectOperations as _;
|
||||
|
||||
const BUCKET: &str = "quota-reset-preservation";
|
||||
const OPERATION: &str = "00000000-0000-0000-0000-000000000002";
|
||||
|
||||
async fn reservation_fixture() -> (tempfile::TempDir, Arc<ECStore>, Uuid, String, Vec<u8>) {
|
||||
let (directory, store) = setup_scanner_cycle_store().await;
|
||||
store
|
||||
.make_bucket(BUCKET, &crate::storage_api::scan::MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the reservation fixture bucket through its owner");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("durable bucket incarnation");
|
||||
assert!(!incarnation.is_nil());
|
||||
let path = format!("config/quota-ledger/{BUCKET}.json");
|
||||
let bytes = serde_json::to_vec(&serde_json::json!({
|
||||
"version": 1,
|
||||
"bucket_incarnation": incarnation,
|
||||
"quota_revision_unix_nanos": 1,
|
||||
"accounted_usage": 100,
|
||||
"reservations": {
|
||||
OPERATION: {
|
||||
"object": "pending-object",
|
||||
"old_size": 0,
|
||||
"new_size": 64,
|
||||
"created_at": 1,
|
||||
"pool_index": 0,
|
||||
"set_index": 0,
|
||||
"commit_started": true
|
||||
}
|
||||
}
|
||||
}))
|
||||
.expect("encode the committed reservation fixture");
|
||||
save_config(store.clone(), &path, bytes.clone())
|
||||
.await
|
||||
.expect("persist reservation bytes through the real storage owner");
|
||||
(directory, store, incarnation, path, bytes)
|
||||
}
|
||||
|
||||
async fn assert_reservation_retained(store: &Arc<ECStore>, path: &str, expected: &[u8], incarnation: Uuid) {
|
||||
let bytes = read_config(store.clone(), path)
|
||||
.await
|
||||
.expect("read the actual reservation ledger");
|
||||
assert_eq!(bytes, expected, "scanner reset must not rewrite the reservation ledger");
|
||||
let ledger: serde_json::Value = serde_json::from_slice(&bytes).expect("persisted ledger JSON");
|
||||
assert_eq!(ledger["version"], 1);
|
||||
assert_eq!(ledger["bucket_incarnation"], incarnation.to_string());
|
||||
assert_eq!(ledger["accounted_usage"], 100);
|
||||
let reservations = ledger["reservations"].as_object().expect("reservation map");
|
||||
assert_eq!(reservations.len(), 1);
|
||||
let pending = &reservations[OPERATION];
|
||||
assert_eq!(pending["old_size"], 0);
|
||||
assert_eq!(pending["new_size"], 64);
|
||||
assert_eq!(pending["commit_started"], true);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("owner incarnation after restart"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reset_preservation_survives_storage_owner_reconstruction() {
|
||||
let (_directory, store, incarnation, path, bytes) = reservation_fixture().await;
|
||||
let reset = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("reset scanner usage through the fenced production entry");
|
||||
assert_eq!(reset.usage_state, "bootstrap-pending");
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
assert!(
|
||||
!Arc::ptr_eq(&store, &restarted),
|
||||
"the assertion must read through a newly constructed ECStore"
|
||||
);
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read reset usage through the reconstructed owner");
|
||||
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("bootstrap usage JSON");
|
||||
assert!(data_usage_info_is_bootstrap_pending(&usage));
|
||||
assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reset_preservation_unknown_protocol_rejects_put_after_restart() {
|
||||
for quota_shape in ["zero", "null", "missing"] {
|
||||
let (_directory, store, incarnation, path, bytes) = reservation_fixture().await;
|
||||
let mut quota = serde_json::json!({
|
||||
"quota_type": "Hard",
|
||||
"reservation_protocol": 2,
|
||||
"reservation_quota": 1024
|
||||
});
|
||||
match quota_shape {
|
||||
"zero" => quota["quota"] = serde_json::json!(0),
|
||||
"null" => quota["quota"] = serde_json::Value::Null,
|
||||
"missing" => {}
|
||||
_ => unreachable!("fixed quota shapes"),
|
||||
}
|
||||
let unknown_quota = serde_json::to_vec("a).expect("unknown but syntactically valid quota protocol");
|
||||
store
|
||||
.update_bucket_metadata_config(BUCKET, rustfs_config::QUOTA_CONFIG_FILE, unknown_quota)
|
||||
.await
|
||||
.expect("persist a future protocol using the real metadata owner");
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(BUCKET)
|
||||
.await
|
||||
.expect("same metadata owner incarnation"),
|
||||
incarnation
|
||||
);
|
||||
reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("scanner reset must not change quota metadata");
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
assert!(!Arc::ptr_eq(&store, &restarted));
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
let mut reader = PutObjReader::from_vec(b"must-not-commit".to_vec());
|
||||
let result = restarted.pools[0].disk_set[0]
|
||||
.put_object(BUCKET, "rejected-object", &mut reader, &ObjectOptions::default())
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("unknown reservation protocol with quota={quota_shape} must not admit a PUT"),
|
||||
};
|
||||
assert!(
|
||||
matches!(error, EcstoreError::PartMissingOrCorrupt),
|
||||
"unexpected protocol rejection: {error}"
|
||||
);
|
||||
let missing = restarted.pools[0].disk_set[0]
|
||||
.get_object_info(BUCKET, "rejected-object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the rejected PUT must not create an object");
|
||||
assert!(
|
||||
matches!(missing, EcstoreError::FileNotFound | EcstoreError::ObjectNotFound(_, _)),
|
||||
"object absence must not be confused with another storage failure: {missing}"
|
||||
);
|
||||
assert_reservation_retained(&restarted, &path, &bytes, incarnation).await;
|
||||
}
|
||||
}
|
||||
@@ -1,973 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::super::cycle_state::cleanup_io_fault;
|
||||
use super::*;
|
||||
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
|
||||
|
||||
async fn seed_cleanup(store: &Arc<ECStore>, state: &str) -> ScannerCycleRecoveryMarker {
|
||||
let cycle = CurrentCycle {
|
||||
current: 3,
|
||||
next: 42,
|
||||
..Default::default()
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
encode_scanner_cycle_state(&cycle, 7).expect("cycle encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist cycle");
|
||||
let usage = DataUsageInfo {
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(41),
|
||||
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("usage encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist usage floor");
|
||||
let marker = ScannerCycleRecoveryMarker {
|
||||
schema_version: 1,
|
||||
primary_revision: "previous-primary".to_string(),
|
||||
generation: 41,
|
||||
leader_epoch: 7,
|
||||
classification: "corrupt".to_string(),
|
||||
first_detected_at_unix_secs: 1,
|
||||
last_attempt_at_unix_secs: 2,
|
||||
retry_count: 1,
|
||||
reason: "operator reset in progress".to_string(),
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
quarantine_path: DATA_USAGE_BLOOM_RECOVERY_PATH.clone(),
|
||||
state: state.to_string(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
serde_json::to_vec(&marker).expect("marker encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist operator marker");
|
||||
marker
|
||||
}
|
||||
|
||||
async fn persisted_state(store: &Arc<ECStore>) -> Vec<(Option<Vec<u8>>, DataUsageCacheRevision)> {
|
||||
let mut state = Vec::new();
|
||||
for path in [
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
] {
|
||||
state.push(
|
||||
read_config_with_revision(store.clone(), path)
|
||||
.await
|
||||
.expect("read exact metadata revision"),
|
||||
);
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
async fn run_disabled_startup(ctx: CancellationToken, store: Arc<ECStore>) {
|
||||
let initialized_before = crate::scanner_runtime_initialized();
|
||||
let cleanup = init_scanner_with_recovery(ctx, store, false).await;
|
||||
if let Some(cleanup) = cleanup {
|
||||
tokio::time::timeout(Duration::from_secs(15), cleanup)
|
||||
.await
|
||||
.expect("finite disabled cleanup attempt")
|
||||
.expect("cleanup task should not panic");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::scanner_runtime_initialized(),
|
||||
initialized_before,
|
||||
"disabled recovery must not start the normal scanner runtime"
|
||||
);
|
||||
}
|
||||
|
||||
fn recovery_intent_request(key: &str, actor: &str) -> ScannerRecoveryIntentRequest {
|
||||
ScannerRecoveryIntentRequest {
|
||||
action: SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD.to_string(),
|
||||
mode: "full-rebuild".to_string(),
|
||||
idempotency_key: key.to_string(),
|
||||
actor_sha256: scanner_recovery_actor_sha256(actor),
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_recovery_intent_record(intent_id: String, state: &str) -> ScannerRecoveryIntentRecord {
|
||||
ScannerRecoveryIntentRecord {
|
||||
schema_version: 1,
|
||||
intent_id,
|
||||
action: SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD.to_string(),
|
||||
mode: "full-rebuild".to_string(),
|
||||
state: state.to_string(),
|
||||
actor_sha256: "a".repeat(64),
|
||||
idempotency_key_sha256: "b".repeat(64),
|
||||
request_sha256: "c".repeat(64),
|
||||
accepted_at_unix_secs: 1,
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_reset_fences(store: &Arc<ECStore>) {
|
||||
let data = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle remains durable");
|
||||
let (cycle, epoch) = decode_scanner_cycle_state(&data).expect("valid preserved cycle");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (3, 42, 8));
|
||||
let usage: DataUsageInfo = serde_json::from_slice(
|
||||
&read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("durable usage fence"),
|
||||
)
|
||||
.expect("valid usage");
|
||||
assert_eq!(usage.scanner_epoch, Some(8));
|
||||
assert_eq!(usage.scanner_cycle, Some(41));
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
async fn assert_rebuilt_reset_fences(store: &Arc<ECStore>, expected_epoch: u64) {
|
||||
let data = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt cycle remains durable");
|
||||
let (cycle, epoch) = decode_scanner_cycle_state(&data).expect("valid rebuilt cycle");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (0, 42, expected_epoch));
|
||||
let usage: DataUsageInfo = serde_json::from_slice(
|
||||
&read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("durable rebuilt usage fence"),
|
||||
)
|
||||
.expect("valid usage");
|
||||
assert_eq!(usage.scanner_epoch, Some(expected_epoch));
|
||||
assert_eq!(usage.scanner_cycle, Some(41));
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_reset_crash_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_SCANNER_RESET_CRASH_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let stage = match std::env::var("RUSTFS_SCANNER_RESET_CRASH_STAGE").as_deref() {
|
||||
Ok("primary-read") => cleanup_io_fault::Stage::PrimaryRead,
|
||||
Ok("primary-write") => cleanup_io_fault::Stage::PrimaryWrite,
|
||||
Ok("usage-fence") => cleanup_io_fault::Stage::UsageFence,
|
||||
other => panic!("unexpected reset crash stage: {other:?}"),
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let store = setup_scanner_cycle_store_at_path(std::path::Path::new(&root), false, 1).await;
|
||||
if stage == cleanup_io_fault::Stage::UsageFence {
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt-cycle".to_vec())
|
||||
.await
|
||||
.expect("force child through reconstruction branch");
|
||||
}
|
||||
let injection = cleanup_io_fault::install(&store, stage, false);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store)
|
||||
.await
|
||||
.expect_err("child should stop at the injected owned I/O boundary");
|
||||
assert!(injection.fired_while_owned());
|
||||
let expected = match stage {
|
||||
cleanup_io_fault::Stage::PrimaryRead => "injected primary read failure",
|
||||
cleanup_io_fault::Stage::PrimaryWrite => "injected primary write failure",
|
||||
cleanup_io_fault::Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
assert!(error.to_string().contains(expected), "{error}");
|
||||
});
|
||||
std::process::exit(77);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_recovers_after_child_process_crash_boundaries() {
|
||||
for (name, expected_rebuilt_epoch) in [("primary-read", None), ("primary-write", None), ("usage-fence", Some(9))] {
|
||||
let temp_dir = tempfile::tempdir().expect("crash fixture directory");
|
||||
let store = setup_scanner_cycle_store_at_path(temp_dir.path(), false, 1).await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
drop(store);
|
||||
|
||||
let status = std::process::Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("scanner::tests::recovery_control::scanner_reset_crash_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_SCANNER_RESET_CRASH_ROOT", temp_dir.path())
|
||||
.env("RUSTFS_SCANNER_RESET_CRASH_STAGE", name)
|
||||
.status()
|
||||
.expect("child crash fixture should start");
|
||||
assert_eq!(status.code(), Some(77), "{name} child did not reach the owned crash boundary");
|
||||
|
||||
let restarted = setup_scanner_cycle_store_at_path(temp_dir.path(), false, 1).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
if let Some(epoch) = expected_rebuilt_epoch {
|
||||
assert_rebuilt_reset_fences(&restarted, epoch).await;
|
||||
} else {
|
||||
assert_reset_fences(&restarted).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_accept_is_durable_and_idempotent() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let request = recovery_intent_request("intent-key-0001", "operator-a");
|
||||
|
||||
let first = accept_scanner_usage_recovery_intent(store.clone(), request.clone())
|
||||
.await
|
||||
.expect("first intent should persist");
|
||||
let record = match first {
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create a durable intent: {other:?}"),
|
||||
};
|
||||
assert_eq!(record.state, "accepted");
|
||||
assert_eq!(record.action, SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD);
|
||||
assert_ne!(record.idempotency_key_sha256, request.idempotency_key);
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let queried = get_scanner_usage_recovery_intent(restarted.clone(), &record.intent_id)
|
||||
.await
|
||||
.expect("persisted intent should read after restart")
|
||||
.expect("persisted intent should exist");
|
||||
assert_eq!(queried, record);
|
||||
|
||||
let replay = accept_scanner_usage_recovery_intent(restarted, request)
|
||||
.await
|
||||
.expect("lost response retry should be idempotent");
|
||||
assert_eq!(replay, ScannerRecoveryIntentAcceptResult::Replayed { record });
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_executor_persists_completed_progress() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let request = recovery_intent_request("intent-key-0001-exec", "operator-a");
|
||||
|
||||
let record = match accept_scanner_usage_recovery_intent(store.clone(), request.clone())
|
||||
.await
|
||||
.expect("intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create a durable intent: {other:?}"),
|
||||
};
|
||||
|
||||
let reset = run_scanner_usage_recovery_intent(CancellationToken::new(), store.clone(), record.intent_id.clone())
|
||||
.await
|
||||
.expect("accepted intent should execute")
|
||||
.expect("accepted intent should produce a reset");
|
||||
assert_eq!(reset.status, "reset");
|
||||
assert_eq!(reset.mode, "full-rebuild");
|
||||
|
||||
let completed = get_scanner_usage_recovery_intent(store.clone(), &record.intent_id)
|
||||
.await
|
||||
.expect("completed intent should read")
|
||||
.expect("completed intent should remain durable");
|
||||
assert_eq!(completed.state, "completed");
|
||||
assert_eq!(completed.intent_id, record.intent_id);
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let replay = accept_scanner_usage_recovery_intent(restarted.clone(), request)
|
||||
.await
|
||||
.expect("lost response retry should return the durable completed intent");
|
||||
assert_eq!(replay, ScannerRecoveryIntentAcceptResult::Replayed { record: completed });
|
||||
let rerun = run_scanner_usage_recovery_intent(CancellationToken::new(), restarted, record.intent_id)
|
||||
.await
|
||||
.expect("terminal intent should not be an execution error");
|
||||
assert!(rerun.is_none(), "completed intent must not start a second reset");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_executor_persists_failed_progress() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let record = match accept_scanner_usage_recovery_intent(
|
||||
store.clone(),
|
||||
recovery_intent_request("intent-key-0001-failed", "operator-a"),
|
||||
)
|
||||
.await
|
||||
.expect("intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create a durable intent: {other:?}"),
|
||||
};
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
ctx.cancel();
|
||||
let error = run_scanner_usage_recovery_intent(ctx, store.clone(), record.intent_id.clone())
|
||||
.await
|
||||
.expect_err("cancelled intent execution should fail");
|
||||
assert!(error.to_string().contains("cancelled"), "{error}");
|
||||
|
||||
let failed = get_scanner_usage_recovery_intent(store, &record.intent_id)
|
||||
.await
|
||||
.expect("failed intent should read")
|
||||
.expect("failed intent should remain durable");
|
||||
assert_eq!(failed.state, "failed");
|
||||
assert_eq!(failed.intent_id, record.intent_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_startup_discovers_only_non_terminal_records() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let accepted = match accept_scanner_usage_recovery_intent(
|
||||
store.clone(),
|
||||
recovery_intent_request("intent-key-0001-startup-accepted", "operator-a"),
|
||||
)
|
||||
.await
|
||||
.expect("accepted startup intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create accepted startup intent: {other:?}"),
|
||||
};
|
||||
let mut running = match accept_scanner_usage_recovery_intent(
|
||||
store.clone(),
|
||||
recovery_intent_request("intent-key-0001-startup-running", "operator-a"),
|
||||
)
|
||||
.await
|
||||
.expect("running startup intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create running startup intent: {other:?}"),
|
||||
};
|
||||
running.state = "running".to_string();
|
||||
let running_path = format!(".usage.v2.recovery-intents/{}.json", running.intent_id);
|
||||
save_config(
|
||||
store.clone(),
|
||||
&running_path,
|
||||
serde_json::to_vec(&running).expect("running intent should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("running intent override should persist");
|
||||
|
||||
for (key, state) in [
|
||||
("intent-key-0001-startup-completed", "completed"),
|
||||
("intent-key-0001-startup-failed", "failed"),
|
||||
] {
|
||||
let mut terminal = match accept_scanner_usage_recovery_intent(store.clone(), recovery_intent_request(key, "operator-a"))
|
||||
.await
|
||||
.expect("terminal startup intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create terminal startup intent: {other:?}"),
|
||||
};
|
||||
terminal.state = state.to_string();
|
||||
let path = format!(".usage.v2.recovery-intents/{}.json", terminal.intent_id);
|
||||
save_config(
|
||||
store.clone(),
|
||||
&path,
|
||||
serde_json::to_vec(&terminal).expect("terminal intent should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("terminal intent override should persist");
|
||||
}
|
||||
save_config(store.clone(), ".usage.v2.recovery-intents/not-a-sha.json", b"{}".to_vec())
|
||||
.await
|
||||
.expect("foreign startup key should persist");
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let replayable = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
|
||||
.await
|
||||
.expect("startup discovery should tolerate terminal and foreign records");
|
||||
let replayable = replayable.into_iter().collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
replayable,
|
||||
[accepted.intent_id, running.intent_id].into_iter().collect(),
|
||||
"startup discovery must only re-drive accepted/running durable intents"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_startup_pages_past_terminal_records() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
for index in 0..70 {
|
||||
let intent_id = format!("{index:064x}");
|
||||
let path = format!(".usage.v2.recovery-intents/{intent_id}.json");
|
||||
let terminal = synthetic_recovery_intent_record(intent_id, "completed");
|
||||
save_config(
|
||||
store.clone(),
|
||||
&path,
|
||||
serde_json::to_vec(&terminal).expect("terminal paging fixture should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("terminal paging fixture should persist");
|
||||
}
|
||||
let pending_id = "f".repeat(64);
|
||||
let pending = synthetic_recovery_intent_record(pending_id.clone(), "accepted");
|
||||
save_config(
|
||||
store.clone(),
|
||||
&format!(".usage.v2.recovery-intents/{pending_id}.json"),
|
||||
serde_json::to_vec(&pending).expect("pending paging fixture should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("pending paging fixture should persist");
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let replayable = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
|
||||
.await
|
||||
.expect("startup discovery should page past terminal records");
|
||||
assert_eq!(replayable, vec![pending_id]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_startup_rejects_corrupt_pending_record() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let record = match accept_scanner_usage_recovery_intent(
|
||||
store.clone(),
|
||||
recovery_intent_request("intent-key-0001-startup-corrupt", "operator-a"),
|
||||
)
|
||||
.await
|
||||
.expect("corrupt startup fixture intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create corrupt startup fixture: {other:?}"),
|
||||
};
|
||||
let path = format!(".usage.v2.recovery-intents/{}.json", record.intent_id);
|
||||
save_config(store.clone(), &path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt intent payload should persist");
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let error = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
|
||||
.await
|
||||
.expect_err("startup discovery must not silently drop corrupt pending intent records");
|
||||
assert!(error.to_string().contains("scanner recovery intent is invalid"), "{error}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_disabled_startup_executes_persisted_non_terminal_intent() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let record = match accept_scanner_usage_recovery_intent(
|
||||
store.clone(),
|
||||
recovery_intent_request("intent-key-0001-startup-exec", "operator-a"),
|
||||
)
|
||||
.await
|
||||
.expect("startup execution intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create startup execution intent: {other:?}"),
|
||||
};
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
|
||||
let completed = get_scanner_usage_recovery_intent(restarted, &record.intent_id)
|
||||
.await
|
||||
.expect("startup-executed intent should read")
|
||||
.expect("startup-executed intent should remain durable");
|
||||
assert_eq!(completed.state, "completed");
|
||||
assert_eq!(completed.intent_id, record.intent_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_rejects_same_namespace_conflict() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let request = recovery_intent_request("intent-key-0002", "operator-a");
|
||||
let record = match accept_scanner_usage_recovery_intent(store.clone(), request.clone())
|
||||
.await
|
||||
.expect("first intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create a durable intent: {other:?}"),
|
||||
};
|
||||
|
||||
let mut unsupported = request.clone();
|
||||
unsupported.mode = "future-mode".to_string();
|
||||
let error = accept_scanner_usage_recovery_intent(store.clone(), unsupported)
|
||||
.await
|
||||
.expect_err("unsupported mode must fail before it can collide with a durable record");
|
||||
assert!(error.to_string().contains("mode is unsupported"));
|
||||
|
||||
let same_key_other_actor = recovery_intent_request("intent-key-0002", "operator-b");
|
||||
let accepted_other_actor = accept_scanner_usage_recovery_intent(store.clone(), same_key_other_actor)
|
||||
.await
|
||||
.expect("another actor owns an independent idempotency namespace");
|
||||
assert!(matches!(accepted_other_actor, ScannerRecoveryIntentAcceptResult::Accepted { .. }));
|
||||
|
||||
let path = format!(".usage.v2.recovery-intents/{}.json", record.intent_id);
|
||||
let mut existing = record.clone();
|
||||
existing.request_sha256 = scanner_recovery_actor_sha256("different-request");
|
||||
save_config(store.clone(), &path, serde_json::to_vec(&existing).expect("mutated record should encode"))
|
||||
.await
|
||||
.expect("mutate durable record");
|
||||
let conflict = accept_scanner_usage_recovery_intent(store, request)
|
||||
.await
|
||||
.expect("same namespace collision should be reported");
|
||||
assert_eq!(
|
||||
conflict,
|
||||
ScannerRecoveryIntentAcceptResult::Conflict {
|
||||
existing: ScannerRecoveryIntentConflict {
|
||||
intent_id: record.intent_id,
|
||||
state: "accepted".to_string(),
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_query_rejects_corrupt_or_unknown_records() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let request = recovery_intent_request("intent-key-0003", "operator-a");
|
||||
let record = match accept_scanner_usage_recovery_intent(store.clone(), request)
|
||||
.await
|
||||
.expect("first intent should persist")
|
||||
{
|
||||
ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
|
||||
other => panic!("first request must create a durable intent: {other:?}"),
|
||||
};
|
||||
let path = format!(".usage.v2.recovery-intents/{}.json", record.intent_id);
|
||||
save_config(store.clone(), &path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt durable record");
|
||||
let error = get_scanner_usage_recovery_intent(store.clone(), &record.intent_id)
|
||||
.await
|
||||
.expect_err("corrupt intent must not decode as absent");
|
||||
assert!(error.to_string().contains("scanner recovery intent is invalid"));
|
||||
let unknown = get_scanner_usage_recovery_intent(store, &scanner_recovery_actor_sha256("missing"))
|
||||
.await
|
||||
.expect("missing intent should read as absent");
|
||||
assert!(unknown.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_recovery_intent_query_rejects_oversized_records() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let intent_id = scanner_recovery_actor_sha256("oversized-intent");
|
||||
let path = format!(".usage.v2.recovery-intents/{intent_id}.json");
|
||||
save_config(store.clone(), &path, vec![b'a'; 16 * 1024 + 1])
|
||||
.await
|
||||
.expect("oversized durable record");
|
||||
|
||||
let error = get_scanner_usage_recovery_intent(store, &intent_id)
|
||||
.await
|
||||
.expect_err("oversized intent must not be materialized");
|
||||
assert!(error.to_string().contains("bounded object size"), "{error}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_reopens_persisted_intent_without_starting_scanner() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
assert_reset_fences(&restarted).await;
|
||||
let completed = persisted_state(&restarted).await;
|
||||
run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
|
||||
assert_eq!(
|
||||
persisted_state(&restarted).await,
|
||||
completed,
|
||||
"a later startup without an intent must not reset again"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_does_not_authorize_blocked_unknown_or_corrupt_markers() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
for kind in ["blocked", "unknown-phase", "future-version", "unknown-field", "corrupt"] {
|
||||
let marker = seed_cleanup(&store, "blocked").await;
|
||||
let mut value = serde_json::to_value(marker).expect("marker value");
|
||||
match kind {
|
||||
"unknown-phase" => value["state"] = "future-phase".into(),
|
||||
"future-version" => value["schema_version"] = 99.into(),
|
||||
"unknown-field" => value["future_hint"] = true.into(),
|
||||
_ => {}
|
||||
}
|
||||
let bytes = if kind == "corrupt" {
|
||||
b"{broken".to_vec()
|
||||
} else {
|
||||
serde_json::to_vec(&value).expect("marker JSON")
|
||||
};
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes)
|
||||
.await
|
||||
.expect("persist rejected marker");
|
||||
let before = persisted_state(&store).await;
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_eq!(persisted_state(&store).await, before, "{kind} must not become an automatic full rescan");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_rechecks_revision_after_waiting_for_leader_lock() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
let mut marker = seed_cleanup(&store, "cleanup-pending").await;
|
||||
let expected = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("intent revision")
|
||||
.1;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold leader ownership");
|
||||
let mut recovery = Box::pin(reset_scanner_cycle_recovery_for_intent(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
Some(expected),
|
||||
None,
|
||||
));
|
||||
assert!(matches!(futures::poll!(&mut recovery), Poll::Pending));
|
||||
marker.state = "blocked".to_string();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
serde_json::to_vec(&marker).expect("replacement marker"),
|
||||
)
|
||||
.await
|
||||
.expect("replace intent while the fixture owns leader lock");
|
||||
let replaced = persisted_state(&store).await;
|
||||
drop(guard);
|
||||
let error = recovery.await.expect_err("old preflight cannot authorize replacement marker");
|
||||
assert!(error.to_string().contains("changed before recovery acquired ownership"));
|
||||
assert_eq!(persisted_state(&store).await, replaced);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_requires_phase_even_when_revision_matches() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "blocked").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let error = reset_scanner_cycle_recovery_for_intent(CancellationToken::new(), store.clone(), Some(before[1].1.clone()), None)
|
||||
.await
|
||||
.expect_err("a matching ETag alone is not operator cleanup authorization");
|
||||
assert!(error.to_string().contains("unchanged cleanup-pending"));
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("explicit v3 core retains full reset authorization");
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_lock_busy_preserves_intent_without_force_unlock() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader");
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("busy leader must block recovery");
|
||||
assert!(error.to_string().contains("leader lock is busy"));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("leader lock is busy"))
|
||||
);
|
||||
assert!(!status.retryable, "disabled startup makes one attempt, not an automatic retry loop");
|
||||
assert!(!guard.is_lock_lost(), "recovery must not revoke the live owner");
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
drop(guard);
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn concurrent_full_rescan_requests_converge_to_one_recovery_fence() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "blocked").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader before concurrent admin resets");
|
||||
|
||||
let first = tokio::spawn(reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()));
|
||||
let second = tokio::spawn(reset_scanner_cycle_recovery(CancellationToken::new(), store.clone()));
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
persisted_state(&store).await,
|
||||
before,
|
||||
"waiting admin reset requests must not mutate state before leader ownership"
|
||||
);
|
||||
|
||||
drop(guard);
|
||||
let (first, second) = tokio::time::timeout(Duration::from_secs(15), async { tokio::join!(first, second) })
|
||||
.await
|
||||
.expect("both admin reset requests should finish after the live owner releases the lock");
|
||||
let first = first.expect("first admin reset task should not panic");
|
||||
let second = second.expect("second admin reset task should not panic");
|
||||
assert!(first.is_ok(), "first admin reset failed: {first:?}");
|
||||
assert!(second.is_ok(), "second admin reset failed: {second:?}");
|
||||
|
||||
assert_reset_fences(&store).await;
|
||||
let completed = persisted_state(&store).await;
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("lost-reply retry after a completed reset should be idempotent");
|
||||
assert_eq!(
|
||||
persisted_state(&store).await,
|
||||
completed,
|
||||
"a later retry must not create a second reset identity or rewrite durable fences"
|
||||
);
|
||||
assert_eq!(scanner_cycle_recovery_status().state, "healthy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_movement_pause_preserves_intent_for_later_startup() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
|
||||
id: "cleanup-movement".to_string(),
|
||||
pool_stats: vec![EcstoreRebalanceStats {
|
||||
participating: true,
|
||||
info: EcstoreRebalanceInfo {
|
||||
start_time: Some(time::OffsetDateTime::now_utc()),
|
||||
status: EcstoreRebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("movement must block reset publication");
|
||||
assert!(error.to_string().contains("blocked by data movement"));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("blocked by data movement"))
|
||||
);
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
*store.rebalance_meta.write().await = None;
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_cancelled_startup_preserves_persisted_work() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let ctx = CancellationToken::new();
|
||||
ctx.cancel();
|
||||
run_disabled_startup(ctx, store.clone()).await;
|
||||
assert_eq!(persisted_state(&store).await, before);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_probe_obeys_cancellation_and_existing_io_deadline() {
|
||||
for cancel in [false, true] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
store.delayed_gets.lock().await.insert(
|
||||
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str()),
|
||||
data_usage_persist_timeout().saturating_add(Duration::from_secs(60)),
|
||||
);
|
||||
let ctx = CancellationToken::new();
|
||||
let mut probe = Box::pin(read_scanner_cleanup_marker(store.clone(), &ctx));
|
||||
assert!(matches!(futures::poll!(&mut probe), Poll::Pending));
|
||||
if cancel {
|
||||
ctx.cancel();
|
||||
} else {
|
||||
advance(data_usage_persist_timeout()).await;
|
||||
}
|
||||
let error = probe.await.expect_err("pending read must be bounded");
|
||||
assert!(error.to_string().contains(if cancel { "cancelled" } else { "timed out" }));
|
||||
assert!(store.put_counts.lock().await.is_empty(), "probe must remain read-only");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn disabled_cleanup_old_observation_cannot_overwrite_a_new_completion() {
|
||||
let original = scanner_cycle_recovery_status();
|
||||
let healthy = ScannerCycleRecoveryStatus {
|
||||
state: "healthy".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
publish_scanner_cleanup_status(healthy.clone(), None).expect("first status version");
|
||||
let old = scanner_cleanup_status_version();
|
||||
publish_scanner_cleanup_status(healthy, None).expect("a newer completion may have identical fields");
|
||||
assert!(
|
||||
publish_scanner_cleanup_status(
|
||||
ScannerCycleRecoveryStatus {
|
||||
state: "cleanup-pending".to_string(),
|
||||
reason: Some("old lock wait failed".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(old)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(scanner_cycle_recovery_status().state, "healthy");
|
||||
publish_scanner_cleanup_status(original, None).expect("restore prior observation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_owned_read_and_write_failures_keep_specific_status() {
|
||||
for (stage, newer_completion) in [
|
||||
(cleanup_io_fault::Stage::PrimaryRead, false),
|
||||
(cleanup_io_fault::Stage::PrimaryWrite, false),
|
||||
(cleanup_io_fault::Stage::PrimaryWrite, true),
|
||||
] {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let injection = cleanup_io_fault::install(&store, stage, newer_completion);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("injected owned I/O boundary");
|
||||
assert!(
|
||||
injection.fired_while_owned(),
|
||||
"fault must occur after real leader ownership and marker validation"
|
||||
);
|
||||
let expected_error = match stage {
|
||||
cleanup_io_fault::Stage::PrimaryRead => "injected primary read failure",
|
||||
cleanup_io_fault::Stage::PrimaryWrite => "injected primary write failure",
|
||||
cleanup_io_fault::Stage::UsageFence => "injected usage fence failure",
|
||||
};
|
||||
assert!(error.to_string().contains(expected_error));
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if newer_completion {
|
||||
assert_eq!(status.state, "healthy", "old error cannot overwrite a newer completion observation");
|
||||
assert!(status.reason.is_none());
|
||||
} else {
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert!(
|
||||
status.reason.as_deref().is_some_and(|reason| reason.contains(expected_error)),
|
||||
"{status:?}"
|
||||
);
|
||||
}
|
||||
let after = persisted_state(&store).await;
|
||||
assert_eq!(after[0], before[0], "failed primary I/O must preserve its prior revision");
|
||||
assert_eq!(after[2], before[2], "failed primary I/O must not advance the usage fence");
|
||||
let marker: ScannerCycleRecoveryMarker =
|
||||
serde_json::from_slice(after[1].0.as_deref().expect("durable marker retained")).expect("valid cleanup marker");
|
||||
assert_eq!(marker.state, "cleanup-pending");
|
||||
drop(injection);
|
||||
run_disabled_startup(CancellationToken::new(), store.clone()).await;
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_invalidated_observation_never_becomes_unconditional() {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let revision = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker revision")
|
||||
.1;
|
||||
let newer = ScannerCycleRecoveryStatus {
|
||||
state: "healthy".to_string(),
|
||||
reason: Some("newer completion owner".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
publish_scanner_cleanup_status(newer.clone(), None).expect("newer observation");
|
||||
let mut invalidated = None;
|
||||
reset_scanner_cycle_recovery_for_intent(CancellationToken::new(), store.clone(), Some(revision), Some(&mut invalidated))
|
||||
.await
|
||||
.expect("metadata cleanup may complete without owning the newest status observation");
|
||||
assert!(invalidated.is_none());
|
||||
assert_eq!(
|
||||
serde_json::to_value(scanner_cycle_recovery_status()).expect("observed status"),
|
||||
serde_json::to_value(newer).expect("newer status")
|
||||
);
|
||||
assert_reset_fences(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_cleanup_later_failure_preserves_rebuilt_primary_status_identity() {
|
||||
for newer_completion in [false, true] {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt-cycle".to_vec())
|
||||
.await
|
||||
.expect("force the full reconstruction branch");
|
||||
let before = persisted_state(&store).await;
|
||||
let injection = cleanup_io_fault::install(&store, cleanup_io_fault::Stage::UsageFence, newer_completion);
|
||||
let error = resume_scanner_cycle_cleanup(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("fail after primary publication");
|
||||
assert!(injection.fired_while_owned());
|
||||
assert!(error.to_string().contains("injected usage fence failure"));
|
||||
let after = persisted_state(&store).await;
|
||||
assert_ne!(after[0].1, before[0].1, "the primary write must actually commit before this failure");
|
||||
let (cycle, epoch) =
|
||||
decode_scanner_cycle_state(after[0].0.as_deref().expect("rebuilt primary")).expect("valid durable reconstruction");
|
||||
assert_eq!((cycle.current, cycle.next, epoch), (0, 42, 8));
|
||||
assert_eq!(after[2], before[2], "usage fence publication was rejected");
|
||||
let marker: ScannerCycleRecoveryMarker =
|
||||
serde_json::from_slice(after[1].0.as_deref().expect("cleanup marker retained")).expect("valid cleanup marker");
|
||||
assert_eq!(marker.state, "cleanup-pending");
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if newer_completion {
|
||||
assert_eq!(status.state, "healthy");
|
||||
assert!(
|
||||
status.reason.is_none(),
|
||||
"old core and outer failure must both retain invalidated ownership"
|
||||
);
|
||||
} else {
|
||||
let DataUsageCacheRevision::Etag(etag) = &after[0].1 else {
|
||||
panic!("rebuilt primary must have a revision");
|
||||
};
|
||||
assert_eq!(status.state, "cleanup-pending");
|
||||
assert_eq!(status.primary_revision.as_deref(), Some(etag.as_str()));
|
||||
assert_eq!(status.generation, Some(cycle.next));
|
||||
assert_eq!(status.leader_epoch, Some(epoch));
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("injected usage fence failure"))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,490 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::super::usage_store::DataUsagePublicationResult;
|
||||
use super::*;
|
||||
use crate::scanner_io::ScannerBucketScanScope;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use sha2::Digest;
|
||||
use std::time::SystemTime;
|
||||
|
||||
const PROOF_BUCKET: &str = "publication-proof-bucket";
|
||||
const PROOF_EPOCH: u64 = 7;
|
||||
const PROOF_CYCLE: u64 = 11;
|
||||
|
||||
async fn settle_namespace_commits(store: &ECStore) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while store.scanner_data_usage_publication_blocked().await {
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fixture namespace commits must settle before collecting complete coverage");
|
||||
}
|
||||
|
||||
async fn complete_candidate(store: &Arc<ECStore>, cycle: u64) -> (crate::scanner_io::ScannerCycleResult, DataUsageInfo) {
|
||||
settle_namespace_commits(store).await;
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let result = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
store.as_ref(),
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx,
|
||||
budget,
|
||||
updates,
|
||||
want_cycle: cycle,
|
||||
leader_epoch: PROOF_EPOCH,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: None,
|
||||
observed_usage_candidate: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
resolved_scope_observer: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real scanner must produce the fixture candidate");
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
let candidate = receiver.recv().await.expect("complete scanner snapshot");
|
||||
assert!(candidate.usage_snapshot_complete);
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(true));
|
||||
assert_eq!(candidate.scanner_cycle, Some(cycle));
|
||||
assert_eq!(candidate.scanner_epoch, Some(PROOF_EPOCH));
|
||||
(result, candidate)
|
||||
}
|
||||
|
||||
async fn candidate_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let (directory, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
store
|
||||
.make_bucket(PROOF_BUCKET, &crate::storage_api::scan::MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create proof fixture bucket through the owner");
|
||||
let mut reader = PutObjReader::from_vec(b"proof".to_vec());
|
||||
store.pools[0].disk_set[0]
|
||||
.put_object(PROOF_BUCKET, "initial", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist fixture object through the owner");
|
||||
crate::scanner_io::record_dirty_usage_bucket(PROOF_BUCKET);
|
||||
settle_namespace_commits(&store).await;
|
||||
(directory, store)
|
||||
}
|
||||
|
||||
async fn read_root(store: &Arc<ECStore>) -> (Option<Vec<u8>>, DataUsageCacheRevision) {
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read actual v2 root bytes and revision")
|
||||
}
|
||||
|
||||
async fn publish_candidate(
|
||||
store: &Arc<ECStore>,
|
||||
scan: &crate::scanner_io::ScannerCycleResult,
|
||||
candidate: DataUsageInfo,
|
||||
baseline: Option<DataUsagePersistBaseline>,
|
||||
) -> DataUsagePublicationResult {
|
||||
let expectation = scan.publication_expectation();
|
||||
assert!(expectation.is_some(), "only a real complete scan may supply the expectation");
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender.send(candidate).await.expect("enqueue the real scan candidate");
|
||||
drop(sender);
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
receiver,
|
||||
Some(PROOF_EPOCH),
|
||||
baseline,
|
||||
ScannerPublicationFence::new(scan.publication_epoch(), None, None).with_ack_expectation(expectation),
|
||||
|| async { None },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_companion_only_does_not_authorize_root_ack() {
|
||||
for companion in [
|
||||
format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let bytes = serde_json::to_vec(&candidate).expect("actual candidate JSON");
|
||||
save_config(store.clone(), &companion, bytes.clone())
|
||||
.await
|
||||
.expect("persist the companion on real disks");
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("companion fallback baseline");
|
||||
assert_eq!(baseline.data.as_deref(), Some(bytes.as_slice()));
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
assert_eq!(read_root(&store).await.0, None);
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate, Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert!(pending, "unacknowledged durable companion work must remain pending");
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"a companion is not the v2 root target"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, (None, DataUsageCacheRevision::Missing));
|
||||
assert_eq!(read_config(store.clone(), &companion).await.expect("companion retained"), bytes);
|
||||
}
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_actual_root_readback_accepts_semantic_json_equivalence() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let canonical = serde_json::to_vec(&candidate).expect("candidate encoding");
|
||||
let mut value = serde_json::to_value(&candidate).expect("candidate value");
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("usage object")
|
||||
.insert("fixture_unknown_field".into(), serde_json::json!({"retained": true}));
|
||||
let different_bytes = serde_json::to_vec_pretty(&value).expect("noncanonical primary JSON");
|
||||
assert_ne!(different_bytes, canonical);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(&different_bytes).expect("semantic primary"),
|
||||
candidate
|
||||
);
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), different_bytes.clone())
|
||||
.await
|
||||
.expect("persist actual primary representation");
|
||||
let before = read_root(&store).await;
|
||||
assert!(matches!(&before.1, DataUsageCacheRevision::Etag(etag) if !etag.is_empty()));
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("real primary revision");
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, proof) = publication.into_parts();
|
||||
let proof = proof.expect("actual primary readback must produce its own root proof");
|
||||
let expected = scan.publication_expectation().expect("real scan expectation");
|
||||
let (etag, raw_digest) = proof.verified_version_for(&expected).expect("proof must bind this candidate");
|
||||
let DataUsageCacheRevision::Etag(expected_etag) = &before.1 else { panic!("actual root ETag") };
|
||||
assert_eq!(etag, expected_etag);
|
||||
let expected_digest: [u8; 32] = sha2::Sha256::digest(&different_bytes).into();
|
||||
assert_eq!(
|
||||
*raw_digest, expected_digest,
|
||||
"proof must record actual bytes, not reserialized candidate bytes"
|
||||
);
|
||||
// Obtain another proof through the same real readback path rather than
|
||||
// fabricating a publication result from the inspected proof above.
|
||||
let publication = publish_candidate(&store, &scan, candidate, None).await;
|
||||
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(acknowledgements.is_empty(), "the single-node fixture has no remote targets");
|
||||
assert!(
|
||||
!crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
"actual root bytes plus a real revision authorize this scan"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, before, "readback must not rewrite unknown fields or whitespace");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_successful_root_cas_authorizes_its_scan() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
let (bytes, revision) = read_root(&store).await;
|
||||
assert!(matches!(revision, DataUsageCacheRevision::Etag(etag) if !etag.is_empty()));
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(&bytes.expect("actual saved root")).expect("root JSON"),
|
||||
candidate
|
||||
);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(!pending);
|
||||
assert!(acknowledgements.is_empty(), "the single-node fixture has no remote targets");
|
||||
assert!(
|
||||
!crate::scanner_io::dirty_usage_buckets_pending(),
|
||||
"the real root CAS must authorize its matching scan"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_observed_candidate_reuse_requires_a_new_root_proof() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let bootstrap = scanner_usage_bootstrap_marker(SystemTime::now(), Some(PROOF_EPOCH));
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&bootstrap).expect("bootstrap root encoding"),
|
||||
)
|
||||
.await
|
||||
.expect("persist authoritative bootstrap root");
|
||||
let (prior_scan, mut observed_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
// Seed a complete but unconverged observation from real scanner coverage;
|
||||
// the production writer attaches its authoritative baseline identity.
|
||||
observed_candidate.usage_snapshot_converged = Some(false);
|
||||
let observation = publish_candidate(&store, &prior_scan, observed_candidate, None).await;
|
||||
let (outcome, proof) = observation.into_parts();
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Saved);
|
||||
assert!(proof.is_none(), "an observational write cannot authorize a root ACK");
|
||||
let (root_before, revision_before) = read_root(&store).await;
|
||||
let observed = read_config(store.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read real persisted observation");
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let (observer, selected) = tokio::sync::oneshot::channel();
|
||||
let scan = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
store.as_ref(),
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx,
|
||||
budget,
|
||||
updates,
|
||||
want_cycle: PROOF_CYCLE + 1,
|
||||
leader_epoch: PROOF_EPOCH,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: root_before.clone().map(Bytes::from),
|
||||
observed_usage_candidate: Some(Bytes::from(observed)),
|
||||
requires_full_scan: false,
|
||||
service_cohort: None,
|
||||
resolved_scope_observer: Some(observer),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("observation-backed scope must run through the real scanner");
|
||||
let scope = selected.await.expect("production resolver decision");
|
||||
assert_eq!(scope.selected_buckets_for_tests(), Some(&HashSet::from([PROOF_BUCKET.to_string()])));
|
||||
assert_eq!(scan.status, ScannerCycleStatus::Complete);
|
||||
let expectation = scan.publication_expectation().expect("reused coverage must be revalidated");
|
||||
assert!(
|
||||
!expectation.same_candidate(&prior_scan.publication_expectation().expect("prior real candidate")),
|
||||
"the observation cannot transfer the previous scan's expectation"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, (root_before, revision_before));
|
||||
assert!(crate::scanner_io::dirty_usage_buckets_pending());
|
||||
let candidate = receiver.recv().await.expect("new validated root candidate");
|
||||
assert_eq!(candidate.scanner_cycle, Some(PROOF_CYCLE + 1));
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(true));
|
||||
let publication = publish_candidate(&store, &scan, candidate, None).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Completed);
|
||||
assert!(!pending);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert!(!crate::scanner_io::dirty_usage_buckets_pending());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_stale_root_cas_keeps_dirty_after_bucket_save() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let mut bucket_cache = DataUsageCache::default();
|
||||
bucket_cache
|
||||
.load(store.pools[0].disk_set[0].clone(), &path_join_buf(&[PROOF_BUCKET, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.expect("real bucket checkpoint must be persisted before root publication");
|
||||
assert!(bucket_cache.info.snapshot_complete);
|
||||
assert_eq!(
|
||||
bucket_cache
|
||||
.checked_flatten(PROOF_BUCKET)
|
||||
.expect("persisted bucket root")
|
||||
.objects,
|
||||
1
|
||||
);
|
||||
let stale_baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("missing root revision");
|
||||
assert_eq!(stale_baseline.revision, DataUsageCacheRevision::Missing);
|
||||
let mut competing = candidate.clone();
|
||||
competing.scanner_epoch = Some(PROOF_EPOCH + 1);
|
||||
competing.scanner_cycle = Some(PROOF_CYCLE + 1);
|
||||
for state in &mut competing.usage_snapshot_set_states {
|
||||
state.scanner_epoch = Some(PROOF_EPOCH + 1);
|
||||
state.scanner_cycle = Some(PROOF_CYCLE + 1);
|
||||
}
|
||||
let competing_bytes = serde_json::to_vec(&competing).expect("competing root");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), competing_bytes.clone())
|
||||
.await
|
||||
.expect("another publisher wins the actual root slot");
|
||||
let before = read_root(&store).await;
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
|
||||
let publication = publish_candidate(&store, &scan, candidate, Some(stale_baseline)).await;
|
||||
assert_eq!(
|
||||
publication.outcome(),
|
||||
DataUsagePersistOutcome::Current,
|
||||
"the old missing revision loses CAS and reconciles the newer root"
|
||||
);
|
||||
let (_, _, acknowledgements) = finalize_scanner_cycle_result(scan, publication);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty);
|
||||
assert_eq!(
|
||||
read_root(&store).await,
|
||||
before,
|
||||
"bucket durability must not authorize replacing the winning root"
|
||||
);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_cannot_transfer_proof_between_real_scan_results() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (first_scan, first_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let (second_scan, second_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
assert_eq!(first_candidate.scanner_epoch, second_candidate.scanner_epoch);
|
||||
assert_eq!(first_candidate.scanner_cycle, second_candidate.scanner_cycle);
|
||||
assert_eq!(first_candidate.objects_total_count, second_candidate.objects_total_count);
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let publication = publish_candidate(&store, &first_scan, first_candidate, Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved);
|
||||
assert!(read_root(&store).await.0.is_some(), "the first scan really published its root");
|
||||
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(second_scan, publication);
|
||||
assert!(pending, "another scan's publication must not finish this scan's dirty maintenance work");
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"same counters and cycle cannot transfer another scan's proof"
|
||||
);
|
||||
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_stale_baseline_cannot_prove_a_replaced_root() {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (first_scan, first_candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&first_candidate).expect("first candidate"),
|
||||
)
|
||||
.await
|
||||
.expect("persist the first candidate on real disks");
|
||||
let stale_baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("capture the genuine first root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let mut reader = PutObjReader::from_vec(b"second".to_vec());
|
||||
store.pools[0].disk_set[0]
|
||||
.put_object(PROOF_BUCKET, "second", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("commit a real namespace change");
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"direct storage writes leave this fixture's scanner hint generation unchanged"
|
||||
);
|
||||
let (_, replacement) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
assert_eq!(first_candidate.scanner_epoch, replacement.scanner_epoch);
|
||||
assert_eq!(first_candidate.scanner_cycle, replacement.scanner_cycle);
|
||||
assert_eq!((first_candidate.objects_total_count, replacement.objects_total_count), (1, 2));
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&replacement).expect("replacement candidate"),
|
||||
)
|
||||
.await
|
||||
.expect("publish the replacement root");
|
||||
let current = read_root(&store).await;
|
||||
assert_ne!(current.1, stale_baseline.revision);
|
||||
|
||||
// The supplied baseline still equals candidate A, but the actual target
|
||||
// now contains B. Compatibility's AlreadyDurable outcome is not proof.
|
||||
let publication = publish_candidate(&store, &first_scan, first_candidate, Some(stale_baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::AlreadyDurable);
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(first_scan, publication);
|
||||
assert!(pending);
|
||||
assert!(acknowledgements.is_empty());
|
||||
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty);
|
||||
assert_eq!(read_root(&store).await, current);
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish() {
|
||||
for mutation in ["remote_ack_target", "publication_epoch", "remote_lease_targets"] {
|
||||
let (_directory, store) = candidate_store().await;
|
||||
let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await;
|
||||
let baseline = read_data_usage_persist_baseline(store.clone())
|
||||
.await
|
||||
.expect("initial root revision");
|
||||
let dirty = crate::scanner_io::dirty_usage_buckets_for_tests();
|
||||
let changed_generation = dirty
|
||||
.get(PROOF_BUCKET)
|
||||
.expect("the real scan has dirty work")
|
||||
.checked_add(1)
|
||||
.expect("bounded fixture generation");
|
||||
let changed_epoch = scan
|
||||
.publication_epoch()
|
||||
.expect("real scan publication epoch")
|
||||
.checked_add(1)
|
||||
.expect("bounded fixture epoch");
|
||||
let publication = publish_candidate(&store, &scan, candidate.clone(), Some(baseline)).await;
|
||||
assert_eq!(publication.outcome(), DataUsagePersistOutcome::Saved, "{mutation}");
|
||||
let root_before = read_root(&store).await;
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<DataUsageInfo>(root_before.0.as_deref().expect("actual saved root"))
|
||||
.expect("persisted root JSON"),
|
||||
candidate,
|
||||
"{mutation}: the original candidate really reached root storage"
|
||||
);
|
||||
|
||||
let changed = match mutation {
|
||||
"remote_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement {
|
||||
host: "proof-peer:9000".to_string(),
|
||||
instance_id: crate::scanner_activity_epoch().to_string(),
|
||||
kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation),
|
||||
}]),
|
||||
"publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)),
|
||||
"remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![(
|
||||
"proof-peer:9000".to_string(),
|
||||
crate::scanner_activity_epoch().to_string(),
|
||||
changed_generation,
|
||||
)]),
|
||||
_ => unreachable!("fixed mutation cases"),
|
||||
};
|
||||
let (_, pending, acknowledgements) = finalize_scanner_cycle_result(changed, publication);
|
||||
assert!(
|
||||
acknowledgements.is_empty(),
|
||||
"{mutation}: the old root proof must not authorize changed ACK work"
|
||||
);
|
||||
assert!(pending, "{mutation}: changed maintenance work must remain pending");
|
||||
assert_eq!(
|
||||
crate::scanner_io::dirty_usage_buckets_for_tests(),
|
||||
dirty,
|
||||
"{mutation}: the changed scan must not clear local dirty work"
|
||||
);
|
||||
assert_eq!(read_root(&store).await, root_before, "{mutation}: the durable original root is retained");
|
||||
}
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user