mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-10 12:19:07 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3a23122104 | |||
| 0b1ebeffec | |||
| 0f9258594c | |||
| 33fc735ee2 | |||
| 509869cfc5 | |||
| 395876ccd0 | |||
| 6083fbbeea | |||
| 42d3267d88 | |||
| 1d35e381b7 | |||
| 0b109b4885 | |||
| 1dea00ffa7 | |||
| addf7a6219 | |||
| 420cf32b2a | |||
| 907410b2f5 | |||
| f10079f1a3 | |||
| a6c3016493 | |||
| 68fb52b752 | |||
| 16f618d41b | |||
| 565d262ede | |||
| 7b010ddd70 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
|
||||
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
|
||||
sha256-darwin=9901a4950099b94d3d2618524d540f1ed4bf6023044903886f98b6c4bf5f30fb
|
||||
sha256-linux=09fd5f48f5556c9fc576357222dccd7ebb0b90b790798f1834db2ec5d67b4952
|
||||
|
||||
@@ -54,6 +54,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
|
||||
@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
|
||||
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
|
||||
# registration budget; retain the queued backup and deadlock assertions.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.default.overrides]]
|
||||
@@ -410,7 +410,7 @@ threads-required = "num-test-threads"
|
||||
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
|
||||
# registration budget; retain the queued backup and deadlock assertions.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
|
||||
@@ -28,11 +28,12 @@ runs:
|
||||
|
||||
- name: Install actionlint
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
|
||||
run: |
|
||||
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
|
||||
curl --fail --location --silent --show-error \
|
||||
--output "$actionlint_dir/actionlint.tar.gz" \
|
||||
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
|
||||
python3 "$ASSET_DOWNLOADER" --repository rhysd/actionlint --tag v1.7.12 --asset actionlint_1.7.12_linux_amd64.tar.gz --output "$actionlint_dir/actionlint.tar.gz" --sha256 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
|
||||
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
|
||||
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
|
||||
rm "$actionlint_dir/actionlint.tar.gz"
|
||||
|
||||
@@ -66,10 +66,7 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
# protobuf-compiler is deliberately absent: the setup-protoc step below
|
||||
# installs 35.1 into the tool cache and prepends it to PATH, so the apt
|
||||
# build (older, and never version-matched) was shadowed on every run and
|
||||
# simply never used.
|
||||
# Use the pinned code generators rather than Ubuntu's older packages.
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
@@ -79,7 +76,9 @@ runs:
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ripgrep
|
||||
ripgrep \
|
||||
python3 \
|
||||
gh
|
||||
|
||||
# musl-gcc is needed by the native musl release leg, and zip/unzip by the
|
||||
# release packaging steps. No CI test lane touches any of them.
|
||||
@@ -88,13 +87,32 @@ runs:
|
||||
shell: bash
|
||||
run: sudo apt-get install -y musl-tools zip unzip
|
||||
|
||||
- name: Install pinned code generators through GitHub API (Linux x64)
|
||||
if: runner.os == 'Linux' && runner.arch == 'X64'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/download_release_asset.py
|
||||
run: |
|
||||
python3 "$ASSET_DOWNLOADER" --repository protocolbuffers/protobuf --tag v35.1 --asset protoc-35.1-linux-x86_64.zip --output "$RUNNER_TEMP/protoc.zip"
|
||||
python3 "$ASSET_DOWNLOADER" --repository google/flatbuffers --tag v25.12.19 --asset 'Linux.flatc.binary.g++-13.zip' --output "$RUNNER_TEMP/flatc.zip"
|
||||
python3 -m zipfile -e "$RUNNER_TEMP/protoc.zip" "$RUNNER_TEMP/protoc"
|
||||
python3 -m zipfile -e "$RUNNER_TEMP/flatc.zip" "$RUNNER_TEMP/flatc"
|
||||
chmod +x "$RUNNER_TEMP/protoc/bin/protoc" "$RUNNER_TEMP/flatc/flatc"
|
||||
"$RUNNER_TEMP/protoc/bin/protoc" --version
|
||||
"$RUNNER_TEMP/flatc/flatc" --version
|
||||
echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/flatc" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install protoc
|
||||
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
||||
with:
|
||||
version: "35.1"
|
||||
repo-token: ${{ github.token }}
|
||||
|
||||
- name: Install flatc
|
||||
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download a named GitHub release asset through its API and verify its digest."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def download(repository, tag, name, output, expected_sha256=None):
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository):
|
||||
raise ValueError("invalid release repository")
|
||||
if not tag or not name or "/" in name or "\\" in name:
|
||||
raise ValueError("invalid release tag or asset name")
|
||||
if expected_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_sha256):
|
||||
raise ValueError("invalid pinned asset digest")
|
||||
result = subprocess.run(
|
||||
["gh", "api", "--hostname", "github.com", f"repos/{repository}/releases/tags/{quote(tag, safe='')}"],
|
||||
check=True, capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
release = json.loads(result.stdout)
|
||||
if not isinstance(release, dict) or release.get("tag_name") != tag:
|
||||
raise ValueError("release metadata does not match the requested tag")
|
||||
assets = release.get("assets")
|
||||
if not isinstance(assets, list) or any(not isinstance(asset, dict) for asset in assets):
|
||||
raise ValueError("invalid release asset inventory")
|
||||
matches = [asset for asset in assets if asset.get("name") == name]
|
||||
if len(matches) != 1:
|
||||
raise ValueError("release must contain exactly one named asset")
|
||||
asset = matches[0]
|
||||
asset_id, size, digest = asset.get("id"), asset.get("size"), asset.get("digest")
|
||||
if type(asset_id) is not int or asset_id <= 0 or type(size) is not int or not 0 < size <= 512 * 1024 * 1024:
|
||||
raise ValueError("invalid release asset identity or size")
|
||||
if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
||||
raise ValueError("release asset has no valid SHA-256 digest")
|
||||
sha256 = digest.removeprefix("sha256:")
|
||||
if expected_sha256 is not None and expected_sha256 != sha256:
|
||||
raise ValueError("release digest differs from the pinned digest")
|
||||
output = Path(output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(dir=output.parent, prefix=".release-asset-", delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
# gh authenticates to the API; its HTTP client strips credentials on
|
||||
# cross-host redirects. No browser URL or token is supplied in argv.
|
||||
subprocess.run(
|
||||
["gh", "api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", f"repos/{repository}/releases/assets/{asset_id}"],
|
||||
check=True, stdout=stream, timeout=300,
|
||||
)
|
||||
with temporary.open("rb") as stream:
|
||||
actual = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
if temporary.stat().st_size != size or actual != sha256:
|
||||
raise ValueError("downloaded release asset failed size or SHA-256 verification")
|
||||
temporary.replace(output)
|
||||
print(f"Verified release asset: {repository} {tag} {name} sha256={actual}")
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repository", required=True)
|
||||
parser.add_argument("--tag", required=True)
|
||||
parser.add_argument("--asset", required=True)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--sha256")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
download(args.repository, args.tag, args.asset, args.output, args.sha256)
|
||||
except (ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.SubprocessError, OSError) as error:
|
||||
print(f"Release asset download failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -193,7 +193,7 @@ jobs:
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest-xlarge","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
|
||||
|
||||
@@ -142,15 +142,23 @@ jobs:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pinned typos through GitHub API
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TYPOS_VERSION: 1.50.2
|
||||
run: |
|
||||
python3 .github/actions/setup/download_release_asset.py --repository crate-ci/typos --tag "v${TYPOS_VERSION}" --asset "typos-v${TYPOS_VERSION}-x86_64-unknown-linux-musl.tar.gz" --output "$RUNNER_TEMP/typos.tar.gz"
|
||||
mkdir -p "$RUNNER_TEMP/typos"
|
||||
tar -xzf "$RUNNER_TEMP/typos.tar.gz" -C "$RUNNER_TEMP/typos"
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
|
||||
run: '"$RUNNER_TEMP/typos/typos" .'
|
||||
|
||||
# Fail early with compile-free checks for every pull request.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -1137,11 +1145,16 @@ jobs:
|
||||
|
||||
- name: Install mc
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
||||
run: |
|
||||
if ! command -v gh >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gh
|
||||
fi
|
||||
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
||||
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
|
||||
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
|
||||
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
||||
chmod +x "$RUNNER_TEMP/mc"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
Generated
+58
-52
@@ -4136,7 +4136,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-compression",
|
||||
@@ -9520,7 +9520,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9680,7 +9680,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"futures",
|
||||
@@ -9702,7 +9702,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9718,7 +9718,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -9733,7 +9733,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9746,7 +9746,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9756,7 +9756,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-connect-symbol-catalog"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hex-simd",
|
||||
"object 0.40.0",
|
||||
@@ -9768,7 +9768,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9782,7 +9782,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9803,7 +9803,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9813,7 +9813,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9837,6 +9837,7 @@ dependencies = [
|
||||
"google-cloud-auth",
|
||||
"google-cloud-storage",
|
||||
"h2",
|
||||
"hashbrown 0.17.1",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"hostname",
|
||||
@@ -9950,7 +9951,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9960,7 +9961,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"base64-simd",
|
||||
@@ -9989,7 +9990,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10030,7 +10031,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal-contracts"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -10040,7 +10041,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10090,7 +10091,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -10103,7 +10104,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -10167,7 +10168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -10193,7 +10194,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -10243,14 +10244,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-license"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10272,7 +10273,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -10296,7 +10297,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -10315,7 +10316,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -10353,7 +10354,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10388,7 +10389,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -10407,7 +10408,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -10424,7 +10425,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -10487,7 +10488,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10518,7 +10519,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
@@ -10581,7 +10582,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10606,7 +10607,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10624,7 +10625,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10670,7 +10671,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10693,7 +10694,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10737,7 +10738,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10745,7 +10746,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10754,20 +10755,25 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"brotli 9.1.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc-fast",
|
||||
"datafusion",
|
||||
"flatbuffers",
|
||||
"flate2",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"lz4",
|
||||
"lz4_flex",
|
||||
"memchr",
|
||||
"metrics",
|
||||
"parking_lot 0.13.0",
|
||||
@@ -10790,7 +10796,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10812,7 +10818,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10863,7 +10869,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner-metrics"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"jiff",
|
||||
@@ -10878,7 +10884,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.21",
|
||||
@@ -10886,7 +10892,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10905,7 +10911,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10920,7 +10926,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10974,7 +10980,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10990,7 +10996,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -11029,7 +11035,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -11066,7 +11072,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64-simd",
|
||||
@@ -11109,7 +11115,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"futures",
|
||||
|
||||
+52
-51
@@ -75,7 +75,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.2"
|
||||
version = "1.0.5"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -92,56 +92,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.2" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.2" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.2" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.2" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.5" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -296,6 +296,7 @@ local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
@@ -184,15 +184,27 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of authenticated RPC signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
|
||||
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
|
||||
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||
/// Nonces remain protected until their signed timestamp is outside the observed wall-clock
|
||||
/// validity window, including after clock rollback. Current-time signatures live about five
|
||||
/// minutes; the maximum accepted forward clock skew can extend that to ten minutes. A body-bound
|
||||
/// v3 request consumes both its v3 and compatibility-v2 nonces, so size for receiver nonce intake,
|
||||
/// not S3 request rate or CPU count.
|
||||
///
|
||||
/// Automatic admission budgets 13% of effective process memory for signed-time
|
||||
/// hash tables, allocator overhead and simultaneous old/new tables during growth.
|
||||
/// The reported count is an absolute UUID payload ceiling, not guaranteed throughput:
|
||||
/// actual byte admission can reject earlier depending on timestamp distribution.
|
||||
/// Tables grow with demand and are released together when their signed time expires.
|
||||
/// CPU count, a historical minimum, and a fixed 32M ceiling do not constrain this budget.
|
||||
/// If memory detection fails, the historical default remains a bounded count fallback.
|
||||
///
|
||||
/// Explicit values remain count ceilings with the historical minimum of 1,048,576
|
||||
/// for configuration compatibility, and bypass automatic byte admission. That
|
||||
/// compatibility floor does not apply to the automatically detected memory budget.
|
||||
/// Zero never disables replay protection.
|
||||
/// Overflow rejects the request without evicting a still-valid identity, increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, and returns resource
|
||||
/// exhaustion rather than invalid credentials. Measure headroom before overriding.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
|
||||
@@ -32,8 +32,9 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
|
||||
|
||||
/// Environment variable name for maximum concurrent disk reads for object operations.
|
||||
/// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
|
||||
/// - Unit: request count (usize).
|
||||
/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
|
||||
/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
|
||||
/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
|
||||
/// A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
|
||||
/// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
|
||||
/// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
|
||||
pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
|
||||
|
||||
@@ -114,10 +114,10 @@ fn staged_commit(metadata: &Value, location: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
async fn location_claim_preserves_other_users_objects_after_drop_and_restart(backing: &str) -> TestResult {
|
||||
async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", backing)];
|
||||
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", "object")];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_TABLE_CATALOG_BACKING", "object")])
|
||||
.await?;
|
||||
env.wait_for_server_ready().await?;
|
||||
@@ -147,39 +147,6 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
|
||||
StatusCode::OK,
|
||||
)
|
||||
.await?;
|
||||
if backing == "durable-strong" {
|
||||
catalog_request(
|
||||
&env,
|
||||
&root,
|
||||
Method::POST,
|
||||
&format!("/iceberg/v1/{bucket}/catalog/warehouse-index/backfill"),
|
||||
None,
|
||||
StatusCode::NO_CONTENT,
|
||||
)
|
||||
.await?;
|
||||
let preflight = catalog_request(
|
||||
&env,
|
||||
&root,
|
||||
Method::GET,
|
||||
&format!("/iceberg/v1/{bucket}/catalog/migration"),
|
||||
None,
|
||||
StatusCode::OK,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(preflight["blockers"], json!([]), "{preflight}");
|
||||
let migration = catalog_request(
|
||||
&env,
|
||||
&root,
|
||||
Method::POST,
|
||||
&format!("/iceberg/v1/{bucket}/catalog/migration"),
|
||||
None,
|
||||
StatusCode::OK,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(migration["ready_to_enable_durable_strong"], json!(true), "{migration}");
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
env.wait_for_server_ready().await?;
|
||||
}
|
||||
admin_create_user(&env, &creator.0, &creator.1).await?;
|
||||
admin_create_user(&env, &reader.0, &reader.1).await?;
|
||||
let create_statement = json!({
|
||||
@@ -341,7 +308,7 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
|
||||
StatusCode::FORBIDDEN,
|
||||
)
|
||||
.await?;
|
||||
if backing == "object" {
|
||||
{
|
||||
catalog_request(
|
||||
&env,
|
||||
&creator,
|
||||
@@ -408,10 +375,5 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
|
||||
location_claim_preserves_other_users_objects_after_drop_and_restart("object").await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_strong_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
|
||||
location_claim_preserves_other_users_objects_after_drop_and_restart("durable-strong").await
|
||||
location_claim_preserves_other_users_objects_after_drop_and_restart().await
|
||||
}
|
||||
|
||||
@@ -121,6 +121,7 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
hashbrown.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
|
||||
@@ -640,17 +640,17 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity,
|
||||
ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
|
||||
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, RpcReplayCacheAdmissionError, S3PeerSys,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement,
|
||||
ScannerPeerActivity, ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||
ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
|
||||
@@ -217,6 +217,8 @@ impl Drop for NotifyOnDrop {
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ListPathRawOptions {
|
||||
/// Retain native listing admission through producer cancellation/draining.
|
||||
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
|
||||
pub disks: Vec<Option<DiskStore>>,
|
||||
pub fallback_disks: Vec<Option<DiskStore>>,
|
||||
pub bucket: String,
|
||||
@@ -227,6 +229,7 @@ pub struct ListPathRawOptions {
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub identifier_tree_leaf: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
/// Deliver every replica to the partial callback, including matching headers.
|
||||
@@ -255,6 +258,7 @@ pub struct ListPathRawOptions {
|
||||
impl Clone for ListPathRawOptions {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
read_budget: self.read_budget.clone(),
|
||||
disks: self.disks.clone(),
|
||||
fallback_disks: self.fallback_disks.clone(),
|
||||
bucket: self.bucket.clone(),
|
||||
@@ -265,6 +269,7 @@ impl Clone for ListPathRawOptions {
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
identifier_tree_leaf: self.identifier_tree_leaf.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
preserve_replica_metadata: self.preserve_replica_metadata,
|
||||
@@ -297,6 +302,7 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
identifier_tree_leaf: opts.identifier_tree_leaf.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
limit: opts.per_disk_limit,
|
||||
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
||||
@@ -389,7 +395,9 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
let read_budget = opts.read_budget.clone();
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let _read_budget = read_budget;
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
@@ -1228,6 +1236,21 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifier_tree_selection_survives_job_clone_and_disk_request() {
|
||||
let request = ListPathRawOptions {
|
||||
path: "catalog/namespaces/".to_string(),
|
||||
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
};
|
||||
let cloned = request.clone();
|
||||
let walk = walk_dir_options(&cloned);
|
||||
assert_eq!(walk.identifier_tree_leaf, request.identifier_tree_leaf);
|
||||
assert_eq!(walk.base_dir, request.path);
|
||||
assert!(walk.recursive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
@@ -1897,6 +1920,42 @@ mod tests {
|
||||
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
|
||||
let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
|
||||
let weak_grant = Arc::downgrade(&grant);
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let stopped = Arc::new(tokio::sync::Notify::new());
|
||||
let worker = tokio::spawn(list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
read_budget: Some(grant),
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("producer entered");
|
||||
assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
|
||||
assert_eq!(semaphore.available_permits(), 0);
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("parent cancelled").is_cancelled());
|
||||
timeout(Duration::from_secs(5), stopped.notified())
|
||||
.await
|
||||
.expect("producer stopped");
|
||||
let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
|
||||
.await
|
||||
.expect("cancelled listing releases all permits")
|
||||
.expect("budget stays open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
|
||||
let result = timeout(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -35,17 +35,17 @@ pub use client::{
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
|
||||
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
RPC_BACKGROUND_ADMISSION_HEADER, RpcReplayCacheAdmissionError, TONIC_RPC_PREFIX, build_auth_headers,
|
||||
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
|
||||
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::cluster::rpc::client::{
|
||||
node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::core::pools::{PoolMeta, PoolMetaWriteState};
|
||||
use crate::core::pools::{PoolMeta, PoolMetaSaveGate, PoolMetaSaveGateWriteGuard};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||
@@ -798,12 +798,12 @@ fn try_acquire_bucket_heal_movement_guard<'a>(
|
||||
}
|
||||
|
||||
async fn acquire_bucket_heal_write_guard<'a>(
|
||||
gate: Option<&'a tokio::sync::Mutex<PoolMetaWriteState>>,
|
||||
) -> Result<Option<tokio::sync::MutexGuard<'a, PoolMetaWriteState>>> {
|
||||
gate: Option<&'a PoolMetaSaveGate>,
|
||||
) -> Result<Option<PoolMetaSaveGateWriteGuard<'a>>> {
|
||||
let Some(gate) = gate else {
|
||||
return Ok(None);
|
||||
};
|
||||
let guard = gate.lock().await;
|
||||
let guard = gate.write().await;
|
||||
guard
|
||||
.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(Error::other))?;
|
||||
@@ -2388,8 +2388,8 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_bucket_heal_refuses_receiver_with_unsafe_pool_metadata() {
|
||||
let gate = tokio::sync::Mutex::new(PoolMetaWriteState::default());
|
||||
gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
||||
let gate = PoolMetaSaveGate::default();
|
||||
gate.write().await.observe_replicas(PoolMetaReplicaState {
|
||||
needs_repair: true,
|
||||
repair_write_safe: false,
|
||||
});
|
||||
|
||||
@@ -376,6 +376,21 @@ struct WalkCopyError {
|
||||
source: io::Error,
|
||||
}
|
||||
|
||||
impl WalkCopyError {
|
||||
fn into_disk_error(self) -> DiskError {
|
||||
// Only a local output write/flush proves the metacache consumer closed.
|
||||
// An identical I/O kind from the remote input remains a transport failure.
|
||||
if matches!(self.reason, RemoteWalkFailureReason::Write | RemoteWalkFailureReason::Flush)
|
||||
&& self.source.kind() == io::ErrorKind::BrokenPipe
|
||||
&& self.source.get_ref().is_none()
|
||||
{
|
||||
DiskError::metacache_output_stream_closed()
|
||||
} else {
|
||||
DiskError::Io(self.source)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn copy_stream_with_buffer<R, W>(
|
||||
reader: &mut R,
|
||||
writer: &mut W,
|
||||
@@ -2096,6 +2111,7 @@ impl RemoteDisk {
|
||||
failure_health_action: FailureHealthAction,
|
||||
) {
|
||||
if let Err(err) = operation_result
|
||||
&& !err.is_metacache_output_stream_closed()
|
||||
&& is_network_like_disk_error(err)
|
||||
{
|
||||
counter!(
|
||||
@@ -4056,7 +4072,7 @@ impl DiskAPI for RemoteDisk {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => {
|
||||
failure_reason = Some(error.reason);
|
||||
return Err(DiskError::Io(error.source));
|
||||
return Err(error.into_disk_error());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9687,6 +9703,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_walk_preserves_local_metacache_consumer_close() {
|
||||
let transport =
|
||||
RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::Data(b"metacache payload".to_vec())]);
|
||||
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
|
||||
let (mut writer, reader) = tokio::io::duplex(1);
|
||||
drop(reader);
|
||||
let error = disk
|
||||
.walk_dir(WalkDirOptions::default(), &mut writer)
|
||||
.await
|
||||
.expect_err("a dropped metacache consumer must stop the producer");
|
||||
assert!(
|
||||
error.is_metacache_output_stream_closed(),
|
||||
"preserve the benign completion marker: {error:?}"
|
||||
);
|
||||
assert_eq!(transport.calls().len(), 1, "local consumer closure is not a remote open retry");
|
||||
assert!(!disk.health.is_faulty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_walk_keeps_remote_read_broken_pipe_as_transport_failure() {
|
||||
let transport = RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::PartialDataThenError {
|
||||
data: b"partial metacache".to_vec(),
|
||||
error: io::ErrorKind::BrokenPipe.into(),
|
||||
}]);
|
||||
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
|
||||
let mut writer = Vec::new();
|
||||
let error = disk
|
||||
.walk_dir(WalkDirOptions::default(), &mut writer)
|
||||
.await
|
||||
.expect_err("an upstream close after partial output is a failed walk");
|
||||
assert!(!error.is_metacache_output_stream_closed());
|
||||
assert!(error.contains_io_error_kind(io::ErrorKind::BrokenPipe));
|
||||
assert_eq!(writer, b"partial metacache");
|
||||
assert_eq!(transport.calls().len(), 1, "partial output must not be replayed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_stream_backpressure_limits_read_ahead_and_is_cancellable() {
|
||||
let mut reader = Cursor::new(vec![5; 4 * DEFAULT_READ_BUFFER_SIZE]);
|
||||
|
||||
@@ -423,11 +423,15 @@ impl RemoteClient {
|
||||
/// peer toward the offline threshold. See issue #4567.
|
||||
fn is_transport_failure(status: &tonic::Status) -> bool {
|
||||
use tonic::Code;
|
||||
// A valid stale-epoch request is rejected before execution. Its signed
|
||||
// response refreshes the peer epoch on the existing healthy channel.
|
||||
let epoch_rejection = status.code() == Code::Unavailable && status.message() == "RPC boot epoch changed";
|
||||
std::error::Error::source(status).is_some()
|
||||
|| matches!(
|
||||
status.code(),
|
||||
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
|
||||
)
|
||||
|| (!epoch_rejection
|
||||
&& matches!(
|
||||
status.code(),
|
||||
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
|
||||
))
|
||||
}
|
||||
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
|
||||
@@ -2827,6 +2831,37 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_boot_epoch_rejections_do_not_open_lock_request_breaker() {
|
||||
let addr = "http://stale-boot-epoch-lock-breaker:9000";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
|
||||
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
let error = client
|
||||
.execute_rpc::<(), _>(
|
||||
"lock",
|
||||
"bucket/new-object",
|
||||
Duration::from_secs(1),
|
||||
async { Err(tonic::Status::unavailable("RPC boot epoch changed")) },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stale epoch still rejects the operation before execution");
|
||||
assert!(error.to_string().contains("RPC boot epoch changed"));
|
||||
}
|
||||
|
||||
let health = lock_peer_health_for_test(addr);
|
||||
assert_eq!(health.request_consecutive_timeouts, 0);
|
||||
assert!(health.request_backoff_until.is_none());
|
||||
assert!(health.last_eviction.is_none());
|
||||
let admission = client
|
||||
.admit_lock_request("lock")
|
||||
.expect("authenticated epoch refresh may proceed immediately");
|
||||
client.release_lock_request("lock", Some(admission));
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn test_remote_client_rpc_timeout_honors_configured_deadline() {
|
||||
|
||||
@@ -628,7 +628,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
||||
let fork = identity_epoch_fork(&fixture).await;
|
||||
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed original failure timestamp should be valid");
|
||||
{
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
state.write_blocked = false;
|
||||
state.block_context = None;
|
||||
@@ -685,7 +685,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
||||
);
|
||||
assert_committed(&fixture).await;
|
||||
{
|
||||
let state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let state = fixture.store.pool_meta_save_gate.write().await;
|
||||
assert!(
|
||||
state.write_blocked,
|
||||
"candidate validation must propagate its permanent latch to the original state"
|
||||
@@ -706,7 +706,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
||||
.expect("permanently blocked recovery should return without mutation"),
|
||||
"ordinary-readable repaired heads must not clear the original known-fork latch"
|
||||
);
|
||||
let state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let state = fixture.store.pool_meta_save_gate.write().await;
|
||||
assert!(state.write_blocked);
|
||||
assert_eq!(
|
||||
state
|
||||
@@ -979,7 +979,7 @@ async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_ra
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
@@ -1036,7 +1036,7 @@ async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
@@ -1110,7 +1110,7 @@ async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
@@ -1199,7 +1199,7 @@ async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order()
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
@@ -1301,7 +1301,7 @@ async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes()
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
@@ -1410,7 +1410,7 @@ async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes()
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
|
||||
+454
-137
File diff suppressed because it is too large
Load Diff
@@ -527,8 +527,8 @@ mod decommission_lock_order_tests {
|
||||
rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(
|
||||
other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(
|
||||
other_store.pool_meta_save_gate.write().await.independent_clone_for_test(),
|
||||
),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
|
||||
@@ -85,6 +85,7 @@ impl ScannerPauseBacklogRetirementPhase {
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
pub set_id: Option<uuid::Uuid>,
|
||||
/// False for retiring members retained only to verify the old commit proof.
|
||||
pub writable: bool,
|
||||
pub data: Option<Vec<u8>>,
|
||||
@@ -140,6 +141,7 @@ async fn read_replica((set, writable): (Arc<SetDisks>, bool)) -> Result<ScannerP
|
||||
let mut replica = ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
set_id: Some(set.persistent_set_id()?),
|
||||
writable,
|
||||
data: None,
|
||||
};
|
||||
@@ -222,6 +224,7 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
.map(|read| ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: read.replica.pool_index,
|
||||
set_index: read.replica.set_index,
|
||||
set_id: read.replica.set_id,
|
||||
writable: read.replica.writable,
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
|
||||
@@ -685,12 +685,8 @@ async fn clear_bucket_usage_memory(bucket: &str, guard: Option<&rustfs_lock::Nam
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
remove_bucket_usage_for_namespace_change(store.as_ref(), bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
|
||||
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
|
||||
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
|
||||
remove_bucket_usage_with_owned_namespace(store, bucket, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
||||
@@ -730,11 +726,37 @@ where
|
||||
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> Result<(), Error> {
|
||||
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
|
||||
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
|
||||
result
|
||||
let owned_store = crate::bucket::metadata_sys::object_store_in(&store.ctx).await?;
|
||||
if !std::ptr::eq(store, owned_store.as_ref()) {
|
||||
return Err(Error::other("usage cleanup requires its own metadata store"));
|
||||
}
|
||||
remove_bucket_usage_with_owned_namespace(owned_store, bucket, guard.cloned()).await
|
||||
}
|
||||
|
||||
async fn remove_bucket_usage_with_owned_namespace(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> Result<(), Error> {
|
||||
let bucket = bucket.to_owned();
|
||||
let ctx = Arc::clone(&store.ctx);
|
||||
// Retain the bucket generation and movement/object guards through every
|
||||
// physical commit even when the namespace request stops waiting.
|
||||
ctx.run_detached_mutation(async move {
|
||||
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(
|
||||
store.as_ref(),
|
||||
&bucket,
|
||||
guard.as_deref(),
|
||||
Some(store.as_ref()),
|
||||
)
|
||||
.await;
|
||||
invalidate_bucket_usage_snapshot_caches(guard.as_deref(), &bucket).await?;
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(Error::from)?
|
||||
}
|
||||
|
||||
async fn invalidate_bucket_usage_snapshot_caches(
|
||||
@@ -849,9 +871,9 @@ where
|
||||
let primary_seed_epoch = match publication_store {
|
||||
Some(publication_store) => Some(
|
||||
publication_store
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
@@ -871,9 +893,9 @@ where
|
||||
let backup_seed_epoch = match publication_store {
|
||||
Some(publication_store) => Some(
|
||||
publication_store
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
@@ -985,9 +1007,9 @@ where
|
||||
let read_epoch = match publication {
|
||||
Some((publication_store, expected_publication_epoch)) => {
|
||||
let epoch = publication_store
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?;
|
||||
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
|
||||
return Err(Error::other("data usage publication epoch changed before snapshot read"));
|
||||
}
|
||||
@@ -1022,9 +1044,8 @@ where
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
|
||||
let publication_guard = match publication {
|
||||
Some((publication_store, expected_publication_epoch)) => {
|
||||
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
|
||||
else {
|
||||
return Err(Error::other("data usage publication is blocked by data movement"));
|
||||
let Some((guard, publication_epoch)) = publication_store.data_usage_namespace_cleanup_guard().await else {
|
||||
return Err(Error::other("data usage namespace cleanup epoch is unavailable"));
|
||||
};
|
||||
if expected_publication_epoch
|
||||
.or(read_epoch)
|
||||
@@ -1036,18 +1057,27 @@ where
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let save_result = store
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut put_data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let save_result = match publication {
|
||||
Some((publication_store, _)) => {
|
||||
publication_store
|
||||
.put_usage_snapshot_for_namespace_cleanup(object, revision.as_deref(), &mut put_data, guard)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
store
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut put_data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
drop(publication_guard);
|
||||
match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
|
||||
Ok(_) => return Ok(()),
|
||||
@@ -3436,7 +3466,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: TokioMutex::new(()),
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
|
||||
@@ -682,15 +682,11 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
||||
return Err(Error::other("ServerNotInitialized"));
|
||||
};
|
||||
|
||||
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
|
||||
for d in all_disks {
|
||||
let pool_info = pools_info.entry(d.pool_index).or_default();
|
||||
let erasure_set = pool_info.entry(d.set_index).or_default();
|
||||
|
||||
if erasure_set.id == 0 {
|
||||
erasure_set.id = d.set_index;
|
||||
collect_pools_info(all_disks, |pool_index, set_index| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let load = load_data_usage_cache(
|
||||
store.pools[d.pool_index as usize].disk_set[d.set_index as usize].as_ref(),
|
||||
store.pools[pool_index as usize].disk_set[set_index as usize].as_ref(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
@@ -698,11 +694,34 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ServerInfoPools,
|
||||
load,
|
||||
);
|
||||
match load.await {
|
||||
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
|
||||
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
}
|
||||
load.await
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn collect_pools_info<F, Fut>(all_disks: &[Disk], mut load_usage: F) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>>
|
||||
where
|
||||
F: FnMut(i32, i32) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<rustfs_data_usage::DataUsageCache>>,
|
||||
{
|
||||
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
|
||||
for d in all_disks {
|
||||
let pool_info = pools_info.entry(d.pool_index).or_default();
|
||||
let erasure_set = match pool_info.entry(d.set_index) {
|
||||
std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
|
||||
std::collections::hash_map::Entry::Vacant(entry) => {
|
||||
let erasure_set = entry.insert(ErasureSetInfo {
|
||||
id: d.set_index,
|
||||
..Default::default()
|
||||
});
|
||||
match load_usage(d.pool_index, d.set_index).await {
|
||||
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
|
||||
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
}
|
||||
erasure_set
|
||||
}
|
||||
};
|
||||
|
||||
erasure_set.raw_capacity += d.total_space;
|
||||
erasure_set.raw_usage += d.used_space;
|
||||
@@ -1241,6 +1260,59 @@ mod tests {
|
||||
assert_eq!(info.deployment_id, expected_deployment_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_usage_is_loaded_once_per_set_including_set_zero() {
|
||||
let mut disks = Vec::new();
|
||||
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
|
||||
disks.extend((0..count).map(|_| Disk {
|
||||
pool_index: pool,
|
||||
set_index: set,
|
||||
used_space: 10,
|
||||
total_space: 100,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
let mut loads = Vec::new();
|
||||
let info = super::collect_pools_info(&disks, |pool, set| {
|
||||
loads.push((pool, set));
|
||||
async move {
|
||||
if pool == 1 && set == 0 {
|
||||
return Err(crate::error::Error::other("usage unavailable"));
|
||||
}
|
||||
let mut cache = rustfs_data_usage::DataUsageCache::default();
|
||||
cache.info.name = DATA_USAGE_ROOT.to_owned();
|
||||
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
|
||||
cache.info.snapshot_complete = true;
|
||||
cache.replace(
|
||||
DATA_USAGE_ROOT,
|
||||
"",
|
||||
rustfs_data_usage::DataUsageEntry {
|
||||
size: 25,
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
Ok(cache)
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("collect mixed successful and unavailable usage");
|
||||
assert_eq!(loads, [(0, 0), (1, 0), (1, 1)]);
|
||||
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
|
||||
let set_info = &info[&pool][&set];
|
||||
assert_eq!(set_info.id, set);
|
||||
assert_eq!(set_info.raw_usage, count * 10);
|
||||
assert_eq!(set_info.raw_capacity, count * 100);
|
||||
if pool == 1 && set == 0 {
|
||||
assert_eq!(set_info.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
} else {
|
||||
assert_eq!(set_info.usage, 25, "logical usage must not multiply by drive count");
|
||||
assert_eq!(set_info.objects_count, 2);
|
||||
assert!(set_info.usage_error.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[tokio::test]
|
||||
async fn local_server_property_includes_runtime_stats_without_endpoint_pools() {
|
||||
|
||||
+465
-114
@@ -562,12 +562,59 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
|
||||
fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<bool> {
|
||||
match std::fs::symlink_metadata(src) {
|
||||
Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup).map(|()| true),
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
Err(std::io::Error::new(ErrorKind::InvalidInput, "multipart transaction source is a symlink"))
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []).map(|()| false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
struct PreparedPartTxnReceipt {
|
||||
src_volume_dir: PathBuf,
|
||||
dst_volume_dir: PathBuf,
|
||||
src_file_path: PathBuf,
|
||||
dst_file_path: PathBuf,
|
||||
dst_meta_path: PathBuf,
|
||||
transaction_path: PathBuf,
|
||||
meta: Bytes,
|
||||
had_old_data: bool,
|
||||
had_old_meta: bool,
|
||||
}
|
||||
|
||||
struct PublishedPartTxnReceipt {
|
||||
current_data_path: PathBuf,
|
||||
current_meta_path: PathBuf,
|
||||
transaction_path: PathBuf,
|
||||
had_old_data: bool,
|
||||
had_old_meta: bool,
|
||||
}
|
||||
|
||||
const PART_TXN_RECEIPTS_PER_SHARD: usize = 64;
|
||||
|
||||
static PREPARED_PART_TXN_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedPartTxnReceipt>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PART_TXN_RECEIPTS_PER_SHARD));
|
||||
|
||||
static PUBLISHED_PART_TXN_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PublishedPartTxnReceipt>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PART_TXN_RECEIPTS_PER_SHARD));
|
||||
|
||||
fn put_prepared_part_txn_receipt(transaction_path: &Path, entry: PreparedPartTxnReceipt) {
|
||||
PREPARED_PART_TXN_RECEIPTS.insert(transaction_path, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_part_txn_receipt(transaction_path: &Path) -> Option<PreparedPartTxnReceipt> {
|
||||
PREPARED_PART_TXN_RECEIPTS.remove(transaction_path)
|
||||
}
|
||||
|
||||
fn put_published_part_txn_receipt(transaction_path: &Path, entry: PublishedPartTxnReceipt) {
|
||||
PUBLISHED_PART_TXN_RECEIPTS.insert(transaction_path, entry);
|
||||
}
|
||||
|
||||
fn take_published_part_txn_receipt(transaction_path: &Path) -> Option<PublishedPartTxnReceipt> {
|
||||
PUBLISHED_PART_TXN_RECEIPTS.remove(transaction_path)
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
@@ -932,6 +979,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
.insert(backup_path.clone());
|
||||
os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await
|
||||
};
|
||||
let restored_backup = restored.is_ok();
|
||||
match restored {
|
||||
Ok(()) => {}
|
||||
Err(DiskError::FileNotFound) if remove_new_metadata => {
|
||||
@@ -961,6 +1009,16 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
os::fsync_dir_with_owner(object_dir, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
if restored_backup {
|
||||
// Renaming two hardlinks to the same inode can leave the backup in place.
|
||||
match os::remove_file_with_owner(&backup_path, namespace_owner.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
if effective_durability(volume).syncs_commit_metadata() {
|
||||
os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
@@ -3160,6 +3218,10 @@ pub(super) fn has_rename_data_test_failpoints(dst_path: &str) -> bool {
|
||||
|| os::has_dir_fsync_test_failures()
|
||||
|| os::file_sync_probe::is_active()
|
||||
|| os::prepared_publication_test_hooks::has_any_hooks()
|
||||
|| !OWNED_FILE_BEFORE_OPEN
|
||||
.lock()
|
||||
.expect("test file open hook lock should not be poisoned")
|
||||
.is_empty()
|
||||
}
|
||||
|
||||
#[cfg(all(not(test), feature = "test-util", target_os = "linux"))]
|
||||
@@ -3745,6 +3807,25 @@ async fn read_dir_entries_with_walk_stall(path: &Path, count: i32, stall: Option
|
||||
const WALK_DIR_BATCH_SIZE: usize = 32;
|
||||
#[cfg(not(test))]
|
||||
const WALK_DIR_BATCH_SIZE: usize = 256;
|
||||
|
||||
async fn read_identifier_tree_entries(path: &Path, leaf: &str, stall: Option<Duration>) -> Result<Vec<String>> {
|
||||
let mut entries = read_dir_with_walk_stall(path.to_path_buf(), stall).await?;
|
||||
let mut names = Vec::new();
|
||||
loop {
|
||||
let (next, batch, eof) = read_dir_entry_batch_with_walk_stall(entries, READ_DIR_BATCH_SIZE, stall).await?;
|
||||
entries = next;
|
||||
// Filter before retaining or sorting the inventory. A report flood
|
||||
// costs directory iteration, not one retained name/metadata read per report.
|
||||
names.extend(batch.into_iter().filter(|name| {
|
||||
name.strip_suffix(SLASH_SEPARATOR)
|
||||
.is_some_and(|component| component == leaf || super::is_identifier_tree_component(component))
|
||||
}));
|
||||
if eof {
|
||||
return Ok(names);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const READ_DIR_BATCH_SIZE: usize = 512;
|
||||
|
||||
async fn read_dir_with_walk_stall(path: PathBuf, stall: Option<Duration>) -> Result<std::fs::ReadDir> {
|
||||
@@ -9220,15 +9301,14 @@ impl LocalDisk {
|
||||
.await;
|
||||
}
|
||||
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let (volume_dir, file_path) = self.io_get_bucket_and_object_path(volume, path)?;
|
||||
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
let object_path = self.get_object_path(volume, path)?;
|
||||
let mut namespace_owner: Option<Arc<dyn Send + Sync>> =
|
||||
Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await);
|
||||
Some(os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await);
|
||||
if opts.recoverable_delete || (opts.undo_write && opts.undo_delete) {
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
namespace_owner = Some(Arc::new((namespace_owner, volume_guard)));
|
||||
@@ -9253,7 +9333,15 @@ impl LocalDisk {
|
||||
let rollback_dir = opts.old_data_dir;
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Some(()) = self
|
||||
.try_fused_recoverable_delete_version(volume, path, &fi, &opts, rollback_dir, namespace_owner.clone())
|
||||
.try_fused_recoverable_delete_version_with_paths(
|
||||
volume,
|
||||
path,
|
||||
&fi,
|
||||
&opts,
|
||||
rollback_dir,
|
||||
namespace_owner.clone(),
|
||||
Some((object_path, volume_dir.clone(), file_path.clone())),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(());
|
||||
@@ -11416,9 +11504,17 @@ impl LocalDisk {
|
||||
admission: Option<multipart_staging::MultipartSourceAdmission>,
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
}
|
||||
let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
|
||||
let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
if fuse_with_admission {
|
||||
self.multipart_index.prepare_record_head(path, &buf).await
|
||||
} else {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
@@ -11434,11 +11530,14 @@ impl LocalDisk {
|
||||
{
|
||||
multipart_source_state::invalidate_active_multipart_source_head(admission.source());
|
||||
}
|
||||
if admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path) {
|
||||
if fuse_with_admission {
|
||||
let precondition = admission.as_ref().map(|a| a.precondition());
|
||||
let root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
|
||||
if let Some(record) = prepared_index {
|
||||
record.run_std()?;
|
||||
}
|
||||
super::fs::create_dir_all_std(&tmp_volume_dir)?;
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&tmp_file_path);
|
||||
@@ -11690,6 +11789,14 @@ impl LocalDisk {
|
||||
W: AsyncWrite + Unpin + Send + 'a,
|
||||
{
|
||||
Box::pin(async move {
|
||||
if opts.identifier_tree_leaf.is_some() {
|
||||
let relative = current
|
||||
.strip_prefix(&opts.base_dir)
|
||||
.ok_or_else(|| DiskError::other("identifier-tree walk escaped its root"))?;
|
||||
if relative.split('/').filter(|component| !component.is_empty()).count() > 512 {
|
||||
return Err(DiskError::other("identifier-tree walk exceeds its directory depth limit"));
|
||||
}
|
||||
}
|
||||
// The part of forward_to below this directory, taken before `current`
|
||||
// loses its trailing slash.
|
||||
let forward_rest = opts
|
||||
@@ -11722,7 +11829,12 @@ impl LocalDisk {
|
||||
// as a single logical read in API semantics.
|
||||
let read_dir_started = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let dir_path_abs = self.io_get_object_path(&opts.bucket, current.trim_start_matches(SLASH_SEPARATOR))?;
|
||||
let read_dir_result = match read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await {
|
||||
let read_dir_result = if let Some(leaf) = opts.identifier_tree_leaf.as_deref() {
|
||||
read_identifier_tree_entries(&dir_path_abs, leaf, stall).await
|
||||
} else {
|
||||
read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await
|
||||
};
|
||||
let read_dir_result = match read_dir_result {
|
||||
Err(err) if err == Error::FileNotFound && !skip_access_checks(&opts.bucket) => {
|
||||
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
|
||||
if let Err(access_err) = cached_access(&volume_dir).await {
|
||||
@@ -12019,6 +12131,16 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
if let Some(leaf) = opts.identifier_tree_leaf.as_deref()
|
||||
&& entry != leaf
|
||||
{
|
||||
// Identifier directories can be namespaces at any depth,
|
||||
// including components named tables or maintenance. Their
|
||||
// own object metadata is outside this selected leaf scan.
|
||||
schedule_dir(&mut dir_stack, format!("{name}/"), true, None, true, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut meta = MetaCacheEntry {
|
||||
name,
|
||||
..Default::default()
|
||||
@@ -12068,7 +12190,8 @@ impl LocalDisk {
|
||||
} else {
|
||||
adaptive_batch_cap
|
||||
};
|
||||
let batch_names = if batch_limit > 1
|
||||
let batch_names = if opts.identifier_tree_leaf.is_none()
|
||||
&& batch_limit > 1
|
||||
&& dir_objes.is_empty()
|
||||
&& dir_stack.iter().all(|(_, _, _, scan_required, _)| !*scan_required)
|
||||
{
|
||||
@@ -12121,6 +12244,11 @@ impl LocalDisk {
|
||||
write_metacache_obj(out, &meta).await?;
|
||||
}
|
||||
|
||||
if opts.identifier_tree_leaf.is_some() {
|
||||
// A dotted leaf cannot contain identifier-tree descendants.
|
||||
continue;
|
||||
}
|
||||
|
||||
let has_namespace_child_candidate = read.has_namespace_child_candidate;
|
||||
let dir_to_skip: HashSet<String> = if has_namespace_child_candidate || !dir_objes.is_empty() {
|
||||
read.data_dirs.into_iter().map(|dir| dir.to_string()).collect()
|
||||
@@ -12167,6 +12295,9 @@ impl LocalDisk {
|
||||
}
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || err == Error::IsNotRegular {
|
||||
if opts.identifier_tree_leaf.is_some() {
|
||||
continue;
|
||||
}
|
||||
adaptive_batch_cap = 1;
|
||||
// NOT an object, append to stack (with slash)
|
||||
// If dirObject, but no metadata (which is unexpected) we skip it.
|
||||
@@ -14307,6 +14438,7 @@ impl DiskAPI for LocalDisk {
|
||||
if bucket == RUSTFS_META_MULTIPART_BUCKET && !paths.is_empty() {
|
||||
let mut cached_parts = Vec::with_capacity(paths.len());
|
||||
let mut all_cached = true;
|
||||
let mut verified_source: Option<(&str, PathBuf)> = None;
|
||||
for path_str in paths {
|
||||
let Some(num) = Path::new(path_str)
|
||||
.file_name()
|
||||
@@ -14318,11 +14450,21 @@ impl DiskAPI for LocalDisk {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
let Ok(source) = self.io_get_object_path(bucket, multipart_staging::upload_scope_root(path_str, true)) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
let scope_root = multipart_staging::upload_scope_root(path_str, true);
|
||||
let (source, head_already_verified) = match verified_source.as_ref() {
|
||||
Some((prev_root, prev_source)) if *prev_root == scope_root => (prev_source, true),
|
||||
_ => {
|
||||
let Ok(source) = self.io_get_object_open_path(bucket, scope_root) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
verified_source = Some((scope_root, source));
|
||||
(&verified_source.as_ref().expect("just initialized").1, false)
|
||||
}
|
||||
};
|
||||
let Some(part) = multipart_source_state::get_active_multipart_source_part(&source, num) else {
|
||||
let Some(part) =
|
||||
multipart_source_state::get_active_multipart_source_part_with_head(source, num, head_already_verified)
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
@@ -14445,8 +14587,8 @@ impl DiskAPI for LocalDisk {
|
||||
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
let source_owner = self.multipart_source_owner(dst_volume, dst_path, true, false, None).await?;
|
||||
let admission = self.multipart_source_admission(dst_volume, dst_path, true, false)?;
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
let (src_volume_dir, src_file_path) = self.io_get_bucket_and_object_path(src_volume, src_path)?;
|
||||
let (dst_volume_dir, dst_file_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
@@ -14454,10 +14596,8 @@ impl DiskAPI for LocalDisk {
|
||||
super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
|
||||
let src_file_path = self.io_get_object_path(src_volume, src_path)?;
|
||||
let dst_file_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
let dst_meta_path = self.io_get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
let dst_meta_path = self.io_get_object_open_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_open_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] {
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
}
|
||||
@@ -14502,9 +14642,13 @@ impl DiskAPI for LocalDisk {
|
||||
if !source.is_file() {
|
||||
return Ok(Err(DiskError::FileAccessDenied));
|
||||
}
|
||||
if transaction_path.exists() {
|
||||
return Ok(Err(DiskError::FileAccessDenied));
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Ok(m) if m.file_type().is_symlink() => return Ok(Err(DiskError::InvalidPath)),
|
||||
Ok(_) => return Ok(Err(DiskError::FileAccessDenied)),
|
||||
Err(err) => return Ok(Err(to_file_error(err).into())),
|
||||
}
|
||||
let mut created_transaction_parent = false;
|
||||
if let Some((integrity, source, destination, expected_len)) = integrity_prep {
|
||||
let stat = match std::fs::symlink_metadata(&source) {
|
||||
Ok(stat) => stat,
|
||||
@@ -14513,6 +14657,7 @@ impl DiskAPI for LocalDisk {
|
||||
if !stat.is_file() || stat.len() != expected_len {
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let presynced_proof = take_presynced_tmp_file(&source);
|
||||
let verify_and_sync = (|| -> std::io::Result<bool> {
|
||||
let mut file = os::open_file_for_sync_std(&source)?;
|
||||
let mut header = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
@@ -14520,7 +14665,7 @@ impl DiskAPI for LocalDisk {
|
||||
if header != integrity.index_header() {
|
||||
return Ok(false);
|
||||
}
|
||||
if durability.syncs_data_shards() && !take_presynced_tmp_file(&source) {
|
||||
if durability.syncs_data_shards() && !presynced_proof {
|
||||
file.sync_all()?;
|
||||
}
|
||||
Ok(true)
|
||||
@@ -14530,7 +14675,13 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(false) => return Ok(Err(DiskError::FileCorrupt)),
|
||||
Err(err) => return Ok(Err(to_file_error(err).into())),
|
||||
}
|
||||
let precondition = admission.as_ref().map(|check| check.precondition());
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
let has_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
|
||||
let has_hooks = false;
|
||||
let precondition = has_hooks
|
||||
.then(|| admission.as_ref().map(|check| check.precondition()))
|
||||
.flatten();
|
||||
// Publish the immutable generation before preparing the part switch.
|
||||
// Rollback retains the old generation; an unreferenced new index is
|
||||
// reclaimed with the upload directory if preparation is interrupted.
|
||||
@@ -14547,12 +14698,13 @@ impl DiskAPI for LocalDisk {
|
||||
) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
created_transaction_parent = same_parent;
|
||||
}
|
||||
|
||||
let Some(transaction_parent) = transaction_path.parent() else {
|
||||
return Ok(Err(DiskError::InvalidPath));
|
||||
};
|
||||
if let Err(err) = std::fs::create_dir_all(transaction_parent) {
|
||||
if !created_transaction_parent && let Err(err) = std::fs::create_dir_all(transaction_parent) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
|
||||
@@ -14560,22 +14712,41 @@ impl DiskAPI for LocalDisk {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
|
||||
let prepare_result = (|| -> std::io::Result<()> {
|
||||
let prepare_result = (|| -> std::io::Result<(bool, bool)> {
|
||||
let old_data_absent = staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT);
|
||||
let old_meta_absent = staging_path.join(PART_TRANSACTION_OLD_META_ABSENT);
|
||||
let had_old_data = snapshot_part_transaction_file(
|
||||
&dst_file_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
)?;
|
||||
let had_old_meta = snapshot_part_transaction_file(
|
||||
&dst_meta_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
&old_data_absent,
|
||||
)?;
|
||||
let had_old_meta = match std::fs::symlink_metadata(&dst_meta_path) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
std::fs::hard_link(&dst_meta_path, staging_path.join(PART_TRANSACTION_OLD_META))?;
|
||||
true
|
||||
}
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "multipart transaction source is a symlink"));
|
||||
}
|
||||
Ok(_) => {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file"));
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
if !had_old_data && std::fs::hard_link(&old_data_absent, &old_meta_absent).is_ok() {
|
||||
false
|
||||
} else {
|
||||
std::fs::write(&old_meta_absent, [])?;
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let new_meta_path = staging_path.join(PART_TRANSACTION_NEW_META);
|
||||
let mut new_meta = std::fs::OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(staging_path.join(PART_TRANSACTION_NEW_META))?;
|
||||
.open(&new_meta_path)?;
|
||||
std::io::Write::write_all(&mut new_meta, &meta)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
new_meta.sync_data()?;
|
||||
@@ -14583,6 +14754,7 @@ impl DiskAPI for LocalDisk {
|
||||
// Windows rejects renaming a directory while one of its children is
|
||||
// still open, even when the child handle shares delete access.
|
||||
drop(new_meta);
|
||||
let _ = std::fs::hard_link(&new_meta_path, staging_path.join(PART_TRANSACTION_PUBLISH_META));
|
||||
let sync_overwrite_backup = durability.syncs_commit_metadata() && (had_old_data || had_old_meta);
|
||||
if sync_overwrite_backup {
|
||||
os::fsync_dir_std(&staging_path)?;
|
||||
@@ -14591,13 +14763,30 @@ impl DiskAPI for LocalDisk {
|
||||
if sync_overwrite_backup {
|
||||
os::fsync_dir_std(transaction_parent)?;
|
||||
}
|
||||
Ok(())
|
||||
Ok((had_old_data, had_old_meta))
|
||||
})();
|
||||
|
||||
if let Err(err) = prepare_result {
|
||||
let _ = remove_dir_all_if_exists(&staging_path);
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
let (had_old_data, had_old_meta) = match prepare_result {
|
||||
Ok(flags) => flags,
|
||||
Err(err) => {
|
||||
let _ = remove_dir_all_if_exists(&staging_path);
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
};
|
||||
put_prepared_part_txn_receipt(
|
||||
&transaction_path,
|
||||
PreparedPartTxnReceipt {
|
||||
src_volume_dir,
|
||||
dst_volume_dir,
|
||||
src_file_path,
|
||||
dst_file_path,
|
||||
dst_meta_path,
|
||||
transaction_path: transaction_path.clone(),
|
||||
meta,
|
||||
had_old_data,
|
||||
had_old_meta,
|
||||
},
|
||||
);
|
||||
|
||||
Ok(Ok(()))
|
||||
})
|
||||
@@ -14616,13 +14805,30 @@ impl DiskAPI for LocalDisk {
|
||||
let source_owner = self.multipart_source_owner(volume, path, true, false, None).await?;
|
||||
let admission = self.multipart_source_admission(volume, path, true, false)?;
|
||||
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
|
||||
self.io_get_bucket_path(volume)?;
|
||||
let current_data_path = self.io_get_object_path(volume, path)?;
|
||||
let current_meta_path = self.io_get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_path(volume, &crate::disk::part_transaction_path(path))?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
let txn_rel_path = crate::disk::part_transaction_path(path);
|
||||
let cached_published = self
|
||||
.io_get_object_open_path(volume, &txn_rel_path)
|
||||
.ok()
|
||||
.and_then(|txn_open| take_published_part_txn_receipt(&txn_open));
|
||||
let (current_data_path, current_meta_path, transaction_path, skip_obsolete_read_on_commit, cached_old_flags) =
|
||||
match cached_published {
|
||||
Some(entry) => (
|
||||
entry.current_data_path,
|
||||
entry.current_meta_path,
|
||||
entry.transaction_path,
|
||||
!entry.had_old_meta,
|
||||
Some((entry.had_old_data, entry.had_old_meta)),
|
||||
),
|
||||
None => {
|
||||
let (_, current_data_path) = self.io_get_bucket_and_object_path(volume, path)?;
|
||||
let current_meta_path = self.io_get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_path(volume, &txn_rel_path)?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
(current_data_path, current_meta_path, transaction_path, false, None)
|
||||
}
|
||||
};
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let part_number = current_data_path
|
||||
@@ -14688,7 +14894,8 @@ impl DiskAPI for LocalDisk {
|
||||
PartTransactionAction::Commit => (PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META),
|
||||
PartTransactionAction::Rollback => (PART_TRANSACTION_OLD_META, PART_TRANSACTION_NEW_META),
|
||||
};
|
||||
if let Ok(obsolete) = std::fs::read(transaction_path.join(obsolete_name))
|
||||
if !(action == PartTransactionAction::Commit && skip_obsolete_read_on_commit && !cfg!(test))
|
||||
&& let Ok(obsolete) = std::fs::read(transaction_path.join(obsolete_name))
|
||||
&& let Ok(obsolete) = ObjectPartInfo::unmarshal(&obsolete)
|
||||
&& let Some(integrity) = obsolete.integrity
|
||||
&& integrity.validate().is_ok()
|
||||
@@ -14720,7 +14927,26 @@ impl DiskAPI for LocalDisk {
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
if let Err(err) = remove_dir_all_if_exists(&cleanup_path) {
|
||||
if let Some((had_old_data, had_old_meta)) = cached_old_flags.filter(|_| action == PartTransactionAction::Commit) {
|
||||
let old_data_name = if had_old_data {
|
||||
PART_TRANSACTION_OLD_DATA
|
||||
} else {
|
||||
PART_TRANSACTION_OLD_DATA_ABSENT
|
||||
};
|
||||
let old_meta_name = if had_old_meta {
|
||||
PART_TRANSACTION_OLD_META
|
||||
} else {
|
||||
PART_TRANSACTION_OLD_META_ABSENT
|
||||
};
|
||||
for name in [PART_TRANSACTION_NEW_META, old_data_name, old_meta_name] {
|
||||
let _ = std::fs::remove_file(cleanup_path.join(name));
|
||||
}
|
||||
if std::fs::remove_dir(&cleanup_path).is_err()
|
||||
&& let Err(err) = remove_dir_all_if_exists(&cleanup_path)
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
} else if let Err(err) = remove_dir_all_if_exists(&cleanup_path) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
Ok(Ok(true))
|
||||
@@ -14754,15 +14980,6 @@ impl DiskAPI for LocalDisk {
|
||||
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
|
||||
pool_meta_native::reject_raw_path(&self.root, src_volume, src_path)?;
|
||||
pool_meta_native::reject_raw_path(&self.root, dst_volume, dst_path)?;
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
super::fs::access_std(&src_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
||||
}
|
||||
|
||||
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
|
||||
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
|
||||
|
||||
@@ -14779,25 +14996,74 @@ impl DiskAPI for LocalDisk {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
let src_file_path = self.io_get_object_path(src_volume, src_path)?;
|
||||
let dst_file_path = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
|
||||
// warn!("rename_part src_file_path:{:?}, dst_file_path:{:?}", &src_file_path, &dst_file_path);
|
||||
|
||||
check_path_length(src_file_path.to_string_lossy().as_ref())?;
|
||||
check_path_length(dst_file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let dst_meta_path = if src_is_dir {
|
||||
None
|
||||
let cached_prepared = if !src_is_dir {
|
||||
let txn_rel = crate::disk::part_transaction_path(dst_path);
|
||||
self.io_get_object_open_path(dst_volume, &txn_rel)
|
||||
.ok()
|
||||
.and_then(|txn_open| take_prepared_part_txn_receipt(&txn_open))
|
||||
.filter(|entry| {
|
||||
entry.meta == meta
|
||||
&& self
|
||||
.io_get_object_open_path(src_volume, src_path)
|
||||
.is_ok_and(|src_open| src_open == entry.src_file_path)
|
||||
})
|
||||
} else {
|
||||
let path = self.io_get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
Some(path)
|
||||
None
|
||||
};
|
||||
let transaction_path = if src_is_dir {
|
||||
None
|
||||
} else {
|
||||
Some(self.io_get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?)
|
||||
|
||||
let (
|
||||
src_volume_dir,
|
||||
dst_volume_dir,
|
||||
src_file_path,
|
||||
dst_file_path,
|
||||
dst_meta_path,
|
||||
transaction_path,
|
||||
preverified_meta_and_old_flags,
|
||||
) = match cached_prepared {
|
||||
Some(entry) => (
|
||||
entry.src_volume_dir,
|
||||
entry.dst_volume_dir,
|
||||
entry.src_file_path,
|
||||
entry.dst_file_path,
|
||||
Some(entry.dst_meta_path),
|
||||
Some(entry.transaction_path),
|
||||
Some((entry.had_old_data, entry.had_old_meta)),
|
||||
),
|
||||
None => {
|
||||
let (src_volume_dir, src_file_path) = self.io_get_bucket_and_object_path(src_volume, src_path)?;
|
||||
let (dst_volume_dir, dst_file_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
super::fs::access_std(&src_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
||||
}
|
||||
|
||||
check_path_length(src_file_path.to_string_lossy().as_ref())?;
|
||||
check_path_length(dst_file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let dst_meta_path = if src_is_dir {
|
||||
None
|
||||
} else {
|
||||
let path = self.io_get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
Some(path)
|
||||
};
|
||||
let transaction_path = if src_is_dir {
|
||||
None
|
||||
} else {
|
||||
Some(self.io_get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?)
|
||||
};
|
||||
(
|
||||
src_volume_dir,
|
||||
dst_volume_dir,
|
||||
src_file_path,
|
||||
dst_file_path,
|
||||
dst_meta_path,
|
||||
transaction_path,
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
// UploadPart is acknowledged once this rename lands, so the part data and
|
||||
@@ -14856,35 +15122,43 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
let transaction_publish_meta = if let Some(transaction_path) = transaction_path.as_ref() {
|
||||
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
|
||||
match std::fs::File::open(&transaction_meta_path) {
|
||||
Ok(mut file) => {
|
||||
let file_len = file.metadata().map_err(to_file_error)?.len();
|
||||
if file_len != meta.len() as u64 {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
let mut expected_meta = Vec::with_capacity(meta.len());
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(&mut file, file_len), &mut expected_meta)
|
||||
.map_err(to_file_error)?;
|
||||
drop(file);
|
||||
if expected_meta.as_slice() != meta.as_ref() {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
|
||||
if !cfg!(test)
|
||||
&& preverified_meta_and_old_flags.is_some()
|
||||
&& publish_meta_path.is_file()
|
||||
&& std::fs::metadata(&transaction_meta_path).is_ok_and(|m| m.is_file() && m.len() == meta.len() as u64)
|
||||
{
|
||||
Some(publish_meta_path)
|
||||
} else {
|
||||
match std::fs::File::open(&transaction_meta_path) {
|
||||
Ok(mut file) => {
|
||||
let file_len = file.metadata().map_err(to_file_error)?.len();
|
||||
if file_len != meta.len() as u64 {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
let mut expected_meta = Vec::with_capacity(meta.len());
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(&mut file, file_len), &mut expected_meta)
|
||||
.map_err(to_file_error)?;
|
||||
drop(file);
|
||||
if expected_meta.as_slice() != meta.as_ref() {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
|
||||
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
|
||||
remove_file_if_exists(&publish_meta_path).map_err(to_file_error)?;
|
||||
std::fs::hard_link(&transaction_meta_path, &publish_meta_path).map_err(to_file_error)?;
|
||||
Some(publish_meta_path)
|
||||
remove_file_if_exists(&publish_meta_path).map_err(to_file_error)?;
|
||||
std::fs::hard_link(&transaction_meta_path, &publish_meta_path).map_err(to_file_error)?;
|
||||
Some(publish_meta_path)
|
||||
}
|
||||
// Old peers know only RenamePart. The new coordinator never
|
||||
// reaches rename unless prepare succeeded, so an absent
|
||||
// transaction directory identifies the rolling-upgrade legacy
|
||||
// path. A present directory without new.meta is corruption.
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => match std::fs::metadata(transaction_path) {
|
||||
Ok(_) => return Err(DiskError::FileCorrupt.into()),
|
||||
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
|
||||
Err(meta_err) => return Err(to_file_error(meta_err)),
|
||||
},
|
||||
Err(err) => return Err(to_file_error(err)),
|
||||
}
|
||||
// Old peers know only RenamePart. The new coordinator never
|
||||
// reaches rename unless prepare succeeded, so an absent
|
||||
// transaction directory identifies the rolling-upgrade legacy
|
||||
// path. A present directory without new.meta is corruption.
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => match std::fs::metadata(transaction_path) {
|
||||
Ok(_) => return Err(DiskError::FileCorrupt.into()),
|
||||
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
|
||||
Err(meta_err) => return Err(to_file_error(meta_err)),
|
||||
},
|
||||
Err(err) => return Err(to_file_error(err)),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
@@ -14894,7 +15168,13 @@ impl DiskAPI for LocalDisk {
|
||||
os::sync_file(&src_file_path).map_err(to_file_error)?;
|
||||
}
|
||||
|
||||
let precondition = admission.as_ref().map(|check| check.precondition());
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
let has_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
|
||||
let has_hooks = false;
|
||||
let precondition = has_hooks
|
||||
.then(|| admission.as_ref().map(|check| check.precondition()))
|
||||
.flatten();
|
||||
|
||||
os::rename_all_with_precondition_and_parent_sync_std(
|
||||
&src_file_path,
|
||||
@@ -14906,17 +15186,19 @@ impl DiskAPI for LocalDisk {
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
let dst_meta = lstat_std(&dst_file_path).map_err(to_file_error)?;
|
||||
if src_is_dir != dst_meta.is_dir() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "dst_type_changed_after_rename",
|
||||
path = ?dst_file_path,
|
||||
"Disk local rename rejected"
|
||||
);
|
||||
return Err(DiskError::FileAccessDenied.into());
|
||||
if cfg!(test) || src_is_dir {
|
||||
let dst_meta = lstat_std(&dst_file_path).map_err(to_file_error)?;
|
||||
if src_is_dir != dst_meta.is_dir() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "dst_type_changed_after_rename",
|
||||
path = ?dst_file_path,
|
||||
"Disk local rename rejected"
|
||||
);
|
||||
return Err(DiskError::FileAccessDenied.into());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(transaction_publish_meta) = transaction_publish_meta {
|
||||
@@ -14947,6 +15229,20 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let (Some((had_old_data, had_old_meta)), Some(txn_path), Some(meta_path)) =
|
||||
(preverified_meta_and_old_flags, transaction_path.as_ref(), dst_meta_path.as_path().into())
|
||||
{
|
||||
put_published_part_txn_receipt(
|
||||
txn_path,
|
||||
PublishedPartTxnReceipt {
|
||||
current_data_path: dst_file_path,
|
||||
current_meta_path: meta_path.to_path_buf(),
|
||||
transaction_path: txn_path.clone(),
|
||||
had_old_data,
|
||||
had_old_meta,
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
@@ -15278,6 +15574,12 @@ impl DiskAPI for LocalDisk {
|
||||
super::lifecycle_test_io::observe(&opts.bucket, None);
|
||||
self.wait_for_startup_cleanup().await;
|
||||
|
||||
if let Some(leaf) = opts.identifier_tree_leaf.as_deref()
|
||||
&& (!opts.recursive || !opts.base_dir.ends_with('/') || !super::is_identifier_tree_leaf(leaf))
|
||||
{
|
||||
return Err(DiskError::other("invalid identifier-tree walk options"));
|
||||
}
|
||||
|
||||
// Callers that do not pin a stall budget inherit the configured default, so
|
||||
// every local walk keeps a liveness bound even when the wrapper-level total
|
||||
// timeout is skipped.
|
||||
@@ -15304,7 +15606,7 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
let mut skip_current_dir_object = false;
|
||||
let mut multipart_dir_to_skip: HashSet<String> = HashSet::new();
|
||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
if opts.identifier_tree_leaf.is_none() && opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
if let Ok(data) = with_walk_stall_timeout(
|
||||
stall,
|
||||
self.read_listing_metadata(
|
||||
@@ -19254,6 +19556,55 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identifier_tree_walk_filters_report_batches_and_bounds_depth() {
|
||||
let dir = tempfile::tempdir().expect("create disk fixture");
|
||||
let root = dir.path().join("catalog/prefix");
|
||||
fs::create_dir_all(root.join("sales"))
|
||||
.await
|
||||
.expect("create identifier directory");
|
||||
fs::create_dir_all(root.join("table-entry.json"))
|
||||
.await
|
||||
.expect("create selected leaf");
|
||||
for index in 0..READ_DIR_BATCH_SIZE * 3 + 1 {
|
||||
fs::create_dir(root.join(format!("report-{index}.json")))
|
||||
.await
|
||||
.expect("create report directory");
|
||||
}
|
||||
let mut names = read_identifier_tree_entries(&root, "table-entry.json", Some(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("all directory batches should be filtered");
|
||||
names.sort();
|
||||
assert_eq!(names, ["sales/", "table-entry.json/"]);
|
||||
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("valid endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk");
|
||||
let options = WalkDirOptions {
|
||||
bucket: "catalog".to_string(),
|
||||
base_dir: "prefix/".to_string(),
|
||||
recursive: true,
|
||||
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mut output = Vec::new();
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
let mut count = 0;
|
||||
let error = disk
|
||||
.scan_dir(
|
||||
format!("prefix/{}", "a/".repeat(513)),
|
||||
String::new(),
|
||||
&options,
|
||||
&mut writer,
|
||||
&mut count,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("excessive depth must fail before filesystem I/O");
|
||||
assert!(error.to_string().contains("directory depth limit"));
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
// #4644: the stall budget bounds a single drive read, never the walk as a
|
||||
// whole. A walk that keeps making progress must survive even when the total
|
||||
// time spent blocked on a slow consumer dwarfs the stall timeout.
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
//! Single-disk object rename publication and rollback. The shared execution core
|
||||
//! retains instrumentation, mutation leases, and commit guards through the syscall.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use super::has_rename_data_test_failpoints;
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -24,8 +22,8 @@ use super::should_fail_local_inline_rollback_hardlink;
|
||||
use super::{
|
||||
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, has_inline_commit_test_failpoints,
|
||||
inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists, remove_dst_base_before_commit,
|
||||
remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
|
||||
has_rename_data_test_failpoints, inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists,
|
||||
remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
|
||||
should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, should_fail_commit_rename,
|
||||
should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
|
||||
};
|
||||
@@ -172,6 +170,44 @@ impl RenameCommitContext<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
fn inline_rollback_backup_path(staging_file_path: &Path) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = if staging_file_path.file_name().and_then(|n| n.to_str()) == Some(STORAGE_FORMAT_FILE) {
|
||||
staging_parent.join(STORAGE_FORMAT_FILE_BACKUP)
|
||||
} else {
|
||||
staging_file_path.with_extension("meta.bkp")
|
||||
};
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
Ok(backup_path)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
fn write_local_inline_rollback_backup_from_bytes(
|
||||
staging_file_path: &Path,
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let backup_path = inline_rollback_backup_path(staging_file_path)?;
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
@@ -179,27 +215,12 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
let backup_path = inline_rollback_backup_path(staging_file_path)?;
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
return write_local_inline_rollback_backup_from_bytes(staging_file_path, old_metadata, sync);
|
||||
} else if cfg!(test) && sync && !old_metadata.is_empty() {
|
||||
// Skip redundant fdatasync on hardlinked rollback backups in production;
|
||||
// keep a Windows-safe sync open in tests.
|
||||
os::open_file_for_sync_std(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
@@ -658,7 +679,14 @@ impl LocalDisk {
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let src_volume_dir = if cfg!(target_os = "linux")
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& skip_access_checks(src_volume)
|
||||
{
|
||||
self.io_root().join(src_volume)
|
||||
} else {
|
||||
self.io_get_bucket_path(src_volume)?
|
||||
};
|
||||
if !skip_access_checks(src_volume)
|
||||
&& let Err(e) = super::super::fs::access_std(&src_volume_dir)
|
||||
{
|
||||
@@ -702,8 +730,24 @@ impl LocalDisk {
|
||||
} else {
|
||||
STORAGE_FORMAT_FILE
|
||||
};
|
||||
let flat_inline_tmp = cfg!(target_os = "linux")
|
||||
&& !no_inline
|
||||
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
|
||||
&& !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let src_file_path = match multipart_publication.as_ref() {
|
||||
Some(publication) => publication.canonical_head(),
|
||||
None if flat_inline_tmp => {
|
||||
self.io_get_object_open_path(src_volume, format!("{src_path}.{source_metadata_name}").as_str())?
|
||||
}
|
||||
None if cfg!(target_os = "linux") => {
|
||||
self.io_get_object_open_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?
|
||||
}
|
||||
None => self.io_get_object_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?,
|
||||
};
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
@@ -714,14 +758,13 @@ impl LocalDisk {
|
||||
.data_dir
|
||||
.map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
|
||||
{
|
||||
let src_data_path = self.io_get_object_path(
|
||||
src_volume,
|
||||
rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(),
|
||||
)?;
|
||||
let dst_data_path = self.io_get_object_path(
|
||||
dst_volume,
|
||||
rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(),
|
||||
)?;
|
||||
let src_rel = rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str());
|
||||
let src_data_path = if cfg!(target_os = "linux") && multipart_publication.is_none() && !fi_healing {
|
||||
self.io_get_object_open_path(src_volume, src_rel.as_str())?
|
||||
} else {
|
||||
self.io_get_object_path(src_volume, src_rel.as_str())?
|
||||
};
|
||||
let dst_data_path = destination_object_path.join(data_dir.trim_matches('/'));
|
||||
|
||||
Some((src_data_path, dst_data_path))
|
||||
} else {
|
||||
@@ -804,7 +847,7 @@ impl LocalDisk {
|
||||
|
||||
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
|
||||
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
|
||||
!no_inline,
|
||||
!no_inline && !flat_inline_tmp,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
@@ -876,7 +919,16 @@ impl LocalDisk {
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
let cleanup_flat_tmp = || {
|
||||
if flat_inline_tmp {
|
||||
let _ = remove_file_if_exists(&src_file_path_owned);
|
||||
}
|
||||
};
|
||||
if let Err(err) = prepared_metadata_source.write_all(&new_dst_buf, false) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
return Err(err);
|
||||
}
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
@@ -884,14 +936,21 @@ impl LocalDisk {
|
||||
let mut staged_rollback_path = None;
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
|
||||
&& (rollback_data_dir.is_some() || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
let backup_path = match create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
) {
|
||||
Ok(path) => path,
|
||||
Err(err) => {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
@@ -920,6 +979,8 @@ impl LocalDisk {
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -928,6 +989,7 @@ impl LocalDisk {
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -944,6 +1006,8 @@ impl LocalDisk {
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -1027,6 +1091,8 @@ impl LocalDisk {
|
||||
None,
|
||||
sync_commit_metadata,
|
||||
) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
@@ -1047,6 +1113,7 @@ impl LocalDisk {
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
cleanup_flat_tmp();
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
@@ -1075,8 +1142,38 @@ impl LocalDisk {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
if sync_commit_metadata {
|
||||
if staged_rollback_path.is_none()
|
||||
&& rollback_data_dir.is_none()
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (!no_inline || rollback_data_dir.is_none())
|
||||
{
|
||||
let fsync_res = (|| -> std::io::Result<()> {
|
||||
if let Some(dst_parent) = dst_file_path_owned.parent() {
|
||||
os::fsync_commit_directory_limited_std(dst_parent, &guard)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = fsync_res {
|
||||
let rollback_ok =
|
||||
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, old_metadata, true)
|
||||
.and_then(|backup_path| {
|
||||
let backup = StagedRollbackBackup(backup_path);
|
||||
let res =
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, Some(&backup));
|
||||
let _ = remove_file_if_exists(backup.path());
|
||||
res
|
||||
})
|
||||
.is_ok();
|
||||
if no_inline {
|
||||
if rollback_ok && let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = remove_dir_all_if_exists(new_data_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
} else if let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
@@ -1087,19 +1184,19 @@ impl LocalDisk {
|
||||
.filter(|_| no_inline)
|
||||
.map(|(_, new_data_path)| new_data_path.as_path()),
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
) {
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
if !flat_inline_tmp && let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
@@ -1364,7 +1461,12 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
match publication_owned.finish_on_thread(
|
||||
sync_commit_metadata,
|
||||
!cfg!(test),
|
||||
staging_cleanup,
|
||||
Some(&directory_guard_owned),
|
||||
) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
|
||||
@@ -856,6 +856,12 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
let prepared_digest = prepared.as_deref().map(digest);
|
||||
let target_digest = target.as_deref().map(digest);
|
||||
let receipt_digest = receipt_bytes.as_deref().map(digest);
|
||||
let finished = finished
|
||||
|| (committed
|
||||
&& prepared.is_none()
|
||||
&& before.is_none()
|
||||
&& target.is_none()
|
||||
&& current_digest == journal.as_ref().and_then(|j| j.target_digest));
|
||||
if let Some(journal) = &journal {
|
||||
if (!finished
|
||||
&& journal.target_digest != target_digest
|
||||
@@ -1113,13 +1119,34 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
Ok((inspection, has_physical_data_dirs_or_markers))
|
||||
}
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
#[derive(Clone)]
|
||||
struct PreparedDeleteReceiptEntry {
|
||||
object_path: PathBuf,
|
||||
bucket_path: PathBuf,
|
||||
file_path: PathBuf,
|
||||
staging_root: PathBuf,
|
||||
journal_raw: Vec<u8>,
|
||||
journal: DeleteRecoveryJournal,
|
||||
sole_transaction: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PublishedDeleteReceiptEntry {
|
||||
object_path: PathBuf,
|
||||
bucket_path: PathBuf,
|
||||
file_path: PathBuf,
|
||||
staging_root: PathBuf,
|
||||
journal_raw: Vec<u8>,
|
||||
}
|
||||
|
||||
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
|
||||
|
||||
static PUBLISHED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PublishedDeleteReceiptEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
|
||||
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
|
||||
}
|
||||
@@ -1128,6 +1155,27 @@ fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceipt
|
||||
PREPARED_DELETE_RECEIPTS.remove(rollback)
|
||||
}
|
||||
|
||||
fn put_published_delete_receipt(rollback: impl AsRef<Path>, entry: PublishedDeleteReceiptEntry) {
|
||||
PUBLISHED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
|
||||
}
|
||||
|
||||
fn take_published_delete_receipt(rollback: &Path) -> Option<PublishedDeleteReceiptEntry> {
|
||||
PUBLISHED_DELETE_RECEIPTS.remove(rollback)
|
||||
}
|
||||
|
||||
fn format_commit_receipt_bytes(journal_raw: &[u8], finished: bool) -> Vec<u8> {
|
||||
let suffix: &[u8] = if finished {
|
||||
b",\"finished\":true}"
|
||||
} else {
|
||||
b",\"finished\":false}"
|
||||
};
|
||||
let mut out = Vec::with_capacity(11 + journal_raw.len() + suffix.len());
|
||||
out.extend_from_slice(b"{\"journal\":");
|
||||
out.extend_from_slice(journal_raw);
|
||||
out.extend_from_slice(suffix);
|
||||
out
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Native MPU source closure has its own conditional protocol. Generic
|
||||
// object recovery must never restore its head or acquire SourceRW under
|
||||
@@ -1781,7 +1829,9 @@ impl LocalDisk {
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
let tx_dir = root.join(transaction.to_string());
|
||||
let _ = take_prepared_delete_receipt(&tx_dir);
|
||||
let _ = take_published_delete_receipt(&tx_dir);
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
@@ -2376,6 +2426,7 @@ impl LocalDisk {
|
||||
barrier(volume, object, transaction, stage, &self.root).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn try_fused_recoverable_delete_version(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2384,6 +2435,21 @@ impl LocalDisk {
|
||||
opts: &DeleteOptions,
|
||||
transaction: Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<()>> {
|
||||
self.try_fused_recoverable_delete_version_with_paths(volume, object, fi, opts, transaction, owner, None)
|
||||
.await
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub(super) async fn try_fused_recoverable_delete_version_with_paths(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
fi: &FileInfo,
|
||||
opts: &DeleteOptions,
|
||||
transaction: Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
prevalidated_paths: Option<(PathBuf, PathBuf, PathBuf)>,
|
||||
) -> Result<Option<()>> {
|
||||
if !opts.recoverable_delete
|
||||
|| opts.undo_write
|
||||
@@ -2396,11 +2462,22 @@ impl LocalDisk {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
self.wait_for_startup_cleanup().await;
|
||||
|
||||
let (_, file_path) = self.io_get_bucket_and_object_path(volume, object)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
let (object_path, bucket_path, file_path) = match prevalidated_paths {
|
||||
Some(paths) => paths,
|
||||
None => {
|
||||
let obj = self.get_object_path(volume, object)?;
|
||||
let (b, f) = self.io_get_bucket_and_object_path(volume, object)?;
|
||||
check_path_length(f.to_string_lossy().as_ref())?;
|
||||
(obj, b, f)
|
||||
}
|
||||
};
|
||||
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
let destination = file_path.join(transaction.to_string());
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let staging_root = if cfg!(target_os = "linux") {
|
||||
self.io_root().join(RUSTFS_META_TMP_BUCKET)
|
||||
} else {
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?
|
||||
};
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let incarnation = opts.delete_recovery_bucket_incarnation;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
@@ -2418,8 +2495,8 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool, bool, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction, destination_missing, old_data_dir_present) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
@@ -2454,7 +2531,11 @@ impl LocalDisk {
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
let mut destination_exists = false;
|
||||
let mut old_data_dir_exists = false;
|
||||
let mut scanned_file_path = false;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
scanned_file_path = true;
|
||||
for (visited, entry) in
|
||||
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
|
||||
{
|
||||
@@ -2467,13 +2548,19 @@ impl LocalDisk {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil()) else {
|
||||
continue;
|
||||
};
|
||||
if id == transaction {
|
||||
destination_exists = true;
|
||||
continue;
|
||||
}
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if old_dir == Some(id) && name_str == id.hyphenated().to_string() {
|
||||
old_data_dir_exists = true;
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
@@ -2532,7 +2619,15 @@ impl LocalDisk {
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
Ok(Some((
|
||||
old_dir,
|
||||
target,
|
||||
raw,
|
||||
journal,
|
||||
sole_transaction,
|
||||
scanned_file_path && !destination_exists,
|
||||
scanned_file_path && old_data_dir_exists,
|
||||
)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2566,27 +2661,33 @@ impl LocalDisk {
|
||||
if sync {
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
Ok(()) => {}
|
||||
if !destination_missing {
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) => return Err(err),
|
||||
},
|
||||
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) => return Err(err),
|
||||
},
|
||||
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
}
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
let old_data_path = old_dir
|
||||
.map(|uuid| file_path.join(uuid.to_string()))
|
||||
.filter(|p| old_data_dir_present || p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(all(test, unix))]
|
||||
fused_prepare_checkpoint::run(&file_path);
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
@@ -2606,10 +2707,18 @@ impl LocalDisk {
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Some(ref target_bytes) = target {
|
||||
// A provisional head must stay outside the published recovery transaction.
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let linked = match std::fs::hard_link(&target_in_rollback, &linked_xl) {
|
||||
Ok(()) => true,
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok()
|
||||
}
|
||||
Err(_) => false,
|
||||
};
|
||||
let write_res = if linked {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
@@ -2653,7 +2762,18 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
put_prepared_delete_receipt(destination, (journal, sole_transaction));
|
||||
put_prepared_delete_receipt(
|
||||
destination,
|
||||
PreparedDeleteReceiptEntry {
|
||||
object_path,
|
||||
bucket_path,
|
||||
file_path,
|
||||
staging_root,
|
||||
journal_raw,
|
||||
journal,
|
||||
sole_transaction,
|
||||
},
|
||||
);
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
@@ -2690,19 +2810,45 @@ impl LocalDisk {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
if transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let object_path = self.get_object_path(volume, object)?;
|
||||
let rollback_open_path = if volume != RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.io_get_object_open_path(volume, object)
|
||||
.ok()
|
||||
.map(|p| p.join(transaction.to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let cached_prepared = rollback_open_path.as_deref().and_then(take_prepared_delete_receipt);
|
||||
let (object_path, bucket_path, root, staging_root, cached_receipt) = match cached_prepared {
|
||||
Some(entry) => (
|
||||
entry.object_path,
|
||||
Some(entry.bucket_path),
|
||||
entry.file_path,
|
||||
entry.staging_root,
|
||||
Some((entry.journal_raw, entry.journal, entry.sole_transaction)),
|
||||
),
|
||||
None => {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let obj_path = self.get_object_path(volume, object)?;
|
||||
(
|
||||
obj_path,
|
||||
None,
|
||||
self.io_get_object_path(volume, object)?,
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let volume_str = volume.to_owned();
|
||||
@@ -2712,23 +2858,38 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
|
||||
if let Some((journal_raw, journal, sole_transaction)) = cached_receipt
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
|
||||
let receipt_bytes = format_commit_receipt_bytes(&journal_raw, false);
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
let mut file = std::fs::File::create(&tmp_receipt)?;
|
||||
std::io::Write::write_all(&mut file, &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
std::fs::rename(&tmp_receipt, &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = std::fs::remove_file(&tmp_receipt);
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
if let Some(bucket_path) = bucket_path {
|
||||
put_published_delete_receipt(
|
||||
rollback,
|
||||
PublishedDeleteReceiptEntry {
|
||||
object_path,
|
||||
bucket_path,
|
||||
file_path: root,
|
||||
staging_root,
|
||||
journal_raw,
|
||||
},
|
||||
);
|
||||
}
|
||||
return Ok(Ok(Some((journal, sole_transaction, true))));
|
||||
}
|
||||
@@ -2851,14 +3012,46 @@ impl LocalDisk {
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
if transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let object_path = self.get_object_path(volume, object)?;
|
||||
let rollback_open_path = if volume != RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.io_get_object_open_path(volume, object)
|
||||
.ok()
|
||||
.map(|p| p.join(transaction.to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let cached_published = rollback_open_path.as_deref().and_then(take_published_delete_receipt);
|
||||
let (object_path, bucket_path, staging_root, root, cached_journal_raw) = match cached_published {
|
||||
Some(entry) => (
|
||||
entry.object_path,
|
||||
entry.bucket_path,
|
||||
entry.staging_root,
|
||||
entry.file_path,
|
||||
Some(entry.journal_raw),
|
||||
),
|
||||
None => {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let obj_path = self.get_object_path(volume, object)?;
|
||||
(
|
||||
obj_path,
|
||||
self.io_get_bucket_path(volume)?,
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
|
||||
self.io_get_object_path(volume, object)?,
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
|
||||
let mut claims: Vec<SnapshotLeaseDeleteClaim> = Vec::with_capacity(journal.data_dirs.len());
|
||||
for data_dir in &journal.data_dirs {
|
||||
@@ -2880,23 +3073,47 @@ impl LocalDisk {
|
||||
claims.push(claim);
|
||||
}
|
||||
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_original = journal.original_digest.is_some();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = journal.clone();
|
||||
let journal_for_receipt = cached_journal_raw.is_none().then(|| journal.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
.map(Path::to_path_buf)
|
||||
.collect();
|
||||
let prune_paths: Vec<_> = if has_target {
|
||||
vec![rollback.clone()]
|
||||
} else {
|
||||
rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
.map(Path::to_path_buf)
|
||||
.collect()
|
||||
};
|
||||
|
||||
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
|
||||
let persist_receipt = |finished: bool| -> std::io::Result<()> {
|
||||
let receipt_bytes = match &cached_journal_raw {
|
||||
Some(raw) => format_commit_receipt_bytes(raw, finished),
|
||||
None => {
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal_for_receipt
|
||||
.clone()
|
||||
.ok_or_else(|| std::io::Error::other("missing journal for receipt"))?,
|
||||
finished,
|
||||
};
|
||||
serde_json::to_vec(&receipt).map_err(std::io::Error::other)?
|
||||
}
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged_receipt.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_receipt.path(), &receipt_path)?;
|
||||
if sync {
|
||||
let _ = os::fsync_dir_std(&rollback);
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
@@ -2950,39 +3167,69 @@ impl LocalDisk {
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if has_target || had_physical_data_dirs {
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal_for_receipt,
|
||||
finished: true,
|
||||
let fast_cleanup = cached_journal_raw.is_some();
|
||||
if fast_cleanup {
|
||||
let backup_or_marker = if has_original {
|
||||
Some(STORAGE_FORMAT_FILE_BACKUP)
|
||||
} else if has_target {
|
||||
Some(DELETE_MARKER_ROLLBACK_FILE)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged_receipt.as_file().sync_data()?;
|
||||
if let Some(name) = backup_or_marker
|
||||
&& let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name)))
|
||||
{
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged_receipt.path(), &receipt_path)?;
|
||||
let _ = std::fs::File::create(rollback.join(FINISHED))?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
for name in [PREPARED, FINISHED].into_iter().chain(has_target.then_some(TARGET)) {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
for name in [PREPARED, TARGET, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
if sync && let Err(err) = os::fsync_dir_std(&rollback) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(&receipt_path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
for dir_path in &prune_paths {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
|
||||
@@ -3105,6 +3352,42 @@ pub(super) async fn restore_barrier(volume: &str, object_dir: &Path, transaction
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod fused_prepare_checkpoint {
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
static HOOKS: LazyLock<Mutex<HashMap<PathBuf, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(super) struct Guard(PathBuf);
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
HOOKS.lock().expect("checkpoint hooks").remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn install(root: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
assert!(
|
||||
HOOKS
|
||||
.lock()
|
||||
.expect("checkpoint hooks")
|
||||
.insert(root.to_path_buf(), Box::new(hook))
|
||||
.is_none()
|
||||
);
|
||||
Guard(root.to_path_buf())
|
||||
}
|
||||
|
||||
pub(super) fn run(root: &Path) {
|
||||
let hook = HOOKS.lock().expect("checkpoint hooks").remove(root);
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -7020,6 +7303,121 @@ mod tests {
|
||||
marker.deleted = false;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_prepare_checkpoint_reopens_before_head_publication() {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let (_temp, disk, mut first, _) = fixture().await;
|
||||
first.version_id = Some(Uuid::new_v4());
|
||||
let mut second = first.clone();
|
||||
second.version_id = Some(Uuid::new_v4());
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(first).expect("add surviving version");
|
||||
meta.add_version(second.clone()).expect("add deleted version");
|
||||
let original = meta.marshal_msg().expect("encode original versions");
|
||||
let (_, root) = disk.io_get_bucket_and_object_path("recovery", "object").expect("object path");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), &original)
|
||||
.await
|
||||
.expect("write versions");
|
||||
|
||||
let (checkpoint_temp, checkpoint_disk, _, _) = fixture().await;
|
||||
let checkpoint_root = checkpoint_temp.path().join("recovery/object");
|
||||
drop(checkpoint_disk);
|
||||
let transaction = Uuid::new_v4();
|
||||
let source_root = root.clone();
|
||||
let copy_root = checkpoint_root.clone();
|
||||
let _capture = fused_prepare_checkpoint::install(&root, move || {
|
||||
let source = source_root.join(transaction.to_string());
|
||||
let destination = copy_root.join(transaction.to_string());
|
||||
std::fs::create_dir(&destination).expect("create checkpoint transaction");
|
||||
let source_head = source_root.join(STORAGE_FORMAT_FILE);
|
||||
let source_head_meta = std::fs::metadata(&source_head).expect("stat source head");
|
||||
let copy_head = copy_root.join(STORAGE_FORMAT_FILE);
|
||||
std::fs::copy(&source_head, ©_head).expect("capture head before publication");
|
||||
for entry in std::fs::read_dir(source).expect("read published preparation") {
|
||||
let entry = entry.expect("prepared file");
|
||||
assert!(entry.file_type().expect("prepared file type").is_file());
|
||||
let dst = destination.join(entry.file_name());
|
||||
if entry.file_name() == STORAGE_FORMAT_FILE_BACKUP {
|
||||
let backup_meta = entry.metadata().expect("stat prepared backup");
|
||||
assert_eq!(
|
||||
(backup_meta.dev(), backup_meta.ino()),
|
||||
(source_head_meta.dev(), source_head_meta.ino()),
|
||||
"fused preparation must hardlink backup to live head"
|
||||
);
|
||||
std::fs::hard_link(©_head, &dst).expect("preserve hardlinked backup");
|
||||
} else {
|
||||
std::fs::copy(entry.path(), dst).expect("capture prepared file");
|
||||
}
|
||||
}
|
||||
});
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
disk.try_fused_recoverable_delete_version("recovery", "object", &second, &opts, transaction, None)
|
||||
.await
|
||||
.expect("finish original preparation"),
|
||||
Some(())
|
||||
);
|
||||
assert!(checkpoint_root.join(transaction.to_string()).join(PREPARED).is_file());
|
||||
let endpoint =
|
||||
Endpoint::try_from(checkpoint_temp.path().to_str().expect("checkpoint path")).expect("checkpoint endpoint");
|
||||
let reopened = LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("reopen checkpoint as a fresh disk");
|
||||
let inspection = reopened
|
||||
.delete_recovery_inspect("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("actual preparation checkpoint must remain recoverable");
|
||||
assert_eq!(inspection.state, DeleteRecoveryState::NewPrepared);
|
||||
let reopened_tx = reopened
|
||||
.io_get_object_path("recovery", &format!("object/{transaction}"))
|
||||
.expect("reopened transaction path");
|
||||
os::fsync_dir_recorder::set_failure(&reopened_tx, ErrorKind::Other);
|
||||
assert!(
|
||||
reopened
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
inspection.fingerprint,
|
||||
DeleteRecoveryAction::Rollback
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"failed transaction directory sync after backup removal must retain journal"
|
||||
);
|
||||
assert!(reopened_tx.join(PREPARED).is_file());
|
||||
assert!(!reopened_tx.join(STORAGE_FORMAT_FILE_BACKUP).exists());
|
||||
let retry_inspection = reopened
|
||||
.delete_recovery_inspect("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect interrupted rollback");
|
||||
assert_eq!(retry_inspection.state, DeleteRecoveryState::NewPrepared);
|
||||
let settled = reopened
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
retry_inspection.fingerprint,
|
||||
DeleteRecoveryAction::Rollback,
|
||||
)
|
||||
.await
|
||||
.expect("roll back interrupted preparation");
|
||||
assert_eq!(settled.state, DeleteRecoveryState::Absent, "same-inode backup must not survive rollback");
|
||||
assert!(!checkpoint_root.join(transaction.to_string()).exists());
|
||||
assert_eq!(
|
||||
fs::read(checkpoint_root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read rolled-back versions"),
|
||||
original
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
|
||||
let (_temp, disk, mut v1, _) = fixture().await;
|
||||
|
||||
@@ -372,29 +372,32 @@ impl IndexOwner {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, openat};
|
||||
let mut directories = vec![self._identity_pin.try_clone()?];
|
||||
for component in [".rustfs.sys", "multipart-index-v1", &sha(bucket)] {
|
||||
let parent = directories.last().ok_or_else(|| io::Error::other("index parent missing"))?;
|
||||
match mkdirat(parent, component, Mode::RWXU | Mode::RWXG | Mode::RWXO) {
|
||||
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
let open_dir = |parent: rustix::fd::BorrowedFd<'_>, component: &str| -> io::Result<rustix::fd::OwnedFd> {
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
match openat(parent, component, flags, Mode::empty()) {
|
||||
Ok(dir) => Ok(dir),
|
||||
Err(rustix::io::Errno::NOENT) => {
|
||||
match mkdirat(parent, component, Mode::RWXU | Mode::RWXG | Mode::RWXO) {
|
||||
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
openat(parent, component, flags, Mode::empty()).map_err(Into::into)
|
||||
}
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
let directory = openat(
|
||||
parent,
|
||||
component,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?;
|
||||
directories.push(std::fs::File::from(directory));
|
||||
}
|
||||
let parent = directories.last().ok_or_else(|| io::Error::other("index parent missing"))?;
|
||||
};
|
||||
use std::os::fd::AsFd;
|
||||
let bucket_sha = sha(bucket);
|
||||
let sys_dir = open_dir(self._identity_pin.as_fd(), ".rustfs.sys")?;
|
||||
let index_dir = open_dir(sys_dir.as_fd(), "multipart-index-v1")?;
|
||||
let parent = open_dir(index_dir.as_fd(), &bucket_sha)?;
|
||||
// The index is boot-local: every start rescans the staging area
|
||||
// before serving it, so a marker lost to a crash is recreated before
|
||||
// it can be consulted. Creation is enough; no fsync of the marker or
|
||||
// its ancestors. An existing marker is touched so pruning sees the
|
||||
// new upload's age, not the previous one's.
|
||||
match openat(
|
||||
parent,
|
||||
&parent,
|
||||
source_root,
|
||||
OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
@@ -409,7 +412,7 @@ impl IndexOwner {
|
||||
// the previous upload's age, which only shortens the prune
|
||||
// grace for this key; index completeness does not depend on it.
|
||||
if let Ok(marker) =
|
||||
openat(parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
|
||||
openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
|
||||
{
|
||||
let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
|
||||
}
|
||||
@@ -426,10 +429,22 @@ impl IndexOwner {
|
||||
os::replace_multipart_source_metadata(&marker, b"1", &self.base, &root)?;
|
||||
os::sync_multipart_owned_record_parents(&self.base.join(".rustfs.sys"), &directory, true, &self.base, &root)?;
|
||||
}
|
||||
self.ensure_current_root()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_current_root(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
let current = {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let metadata = std::fs::symlink_metadata(&*self.root_path)?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(io::Error::other("multipart index physical root is not a directory"));
|
||||
}
|
||||
let mut inode = [0; 16];
|
||||
inode[..8].copy_from_slice(&metadata.ino().to_le_bytes());
|
||||
(metadata.dev(), inode)
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let current = os::PublicationRoot::new(&self.root_path)?.physical_identity()?;
|
||||
if current != self.physical {
|
||||
self.disabled.store(true, Ordering::Release);
|
||||
@@ -532,13 +547,10 @@ impl IndexOwner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let Some(upload) = path
|
||||
pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
|
||||
let upload = path
|
||||
.strip_suffix("/xl.meta")
|
||||
.filter(|upload| canonical_upload(upload).is_some())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
.filter(|upload| canonical_upload(upload).is_some())?;
|
||||
let info = match get_file_info(
|
||||
bytes,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
@@ -553,10 +565,33 @@ impl IndexOwner {
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
self.disabled.store(true, Ordering::Release);
|
||||
return Ok(());
|
||||
return None;
|
||||
}
|
||||
};
|
||||
self.record_info(upload, &info).await
|
||||
let marker = info
|
||||
.metadata
|
||||
.get(crate::disk::multipart_index::RUSTFS_MULTIPART_BUCKET_KEY)
|
||||
.map(|bucket| {
|
||||
self.base
|
||||
.join(INDEX)
|
||||
.join(sha(bucket))
|
||||
.join(upload.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| self.base.join(INDEX));
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
Some(PreparedHeadRecord {
|
||||
owner: Arc::clone(self),
|
||||
upload: upload.to_owned(),
|
||||
info,
|
||||
_lease: lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let Some(prepared) = self.prepare_record_head(path, bytes).await else {
|
||||
return Ok(());
|
||||
};
|
||||
prepared.run().await
|
||||
}
|
||||
|
||||
async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
|
||||
@@ -678,6 +713,37 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHeadRecord {
|
||||
owner: Arc<IndexOwner>,
|
||||
upload: String,
|
||||
info: FileInfo,
|
||||
_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl PreparedHeadRecord {
|
||||
pub(super) fn run_std(self) -> io::Result<()> {
|
||||
let Self {
|
||||
owner,
|
||||
upload,
|
||||
info,
|
||||
_lease,
|
||||
} = self;
|
||||
match owner.record(&upload, &info) {
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidData => {
|
||||
owner.disabled.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(self) -> Result<()> {
|
||||
let lease = Arc::clone(&self._lease);
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(in crate::disk) async fn wait_for_multipart_index_bootstrap(&self) {
|
||||
|
||||
@@ -428,7 +428,6 @@ impl LocalDisk {
|
||||
let root = store.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, Some(store.lease.clone())).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
permit.validate()?;
|
||||
os::rename_multipart_source_owned_std(&source, &destination, &base, &root, || permit.validate())
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -295,10 +295,17 @@ pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: By
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
if FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) != Some(expected_head_stamp) {
|
||||
return false;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if source.join(STATE).exists()
|
||||
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
|| source.join(".mpu-completions").exists()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
@@ -375,7 +382,11 @@ pub(super) fn finish_active_multipart_part_transaction(
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
pub(super) fn get_active_multipart_source_part_with_head(
|
||||
source: &Path,
|
||||
part_number: usize,
|
||||
head_already_verified: bool,
|
||||
) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
@@ -386,7 +397,7 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
if !head_already_verified && !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
@@ -609,25 +620,45 @@ impl SourceStore {
|
||||
raw: &[u8],
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
Self::persist_state_with_guard_std(source, base, root, raw, terminal, sync_source_before, None)
|
||||
}
|
||||
fn persist_state_with_guard_std(
|
||||
source: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
raw: &[u8],
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
directory_guard: Option<&os::RenameCommitGuard>,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
if sync_source_before {
|
||||
#[cfg(unix)]
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| IoError::other("empty source directory guard"))?);
|
||||
os::fsync_directory_handle_std(source, Some(&opened))?;
|
||||
}
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
{
|
||||
let opened = std::fs::File::from(
|
||||
_pinned
|
||||
.last()
|
||||
.ok_or_else(|| IoError::other("empty source directory guard"))?
|
||||
.try_clone()?,
|
||||
);
|
||||
os::fsync_directory_handle_std(source, Some(&opened))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
os::fsync_dir_std(source)?;
|
||||
}
|
||||
if let Some(path) = terminal {
|
||||
os::replace_multipart_source_metadata(path, raw, base, root)?;
|
||||
}
|
||||
os::replace_multipart_source_metadata(&source.join(STATE), raw, base, root)
|
||||
os::replace_multipart_source_metadata_with_guard(&source.join(STATE), raw, base, root, directory_guard)
|
||||
}
|
||||
async fn write_owned_backup_and_persist(&self, owner: Uuid, target: bool, bytes: &[u8], state: &SourceState) -> Result<()> {
|
||||
if owner.is_nil() {
|
||||
@@ -724,17 +755,39 @@ impl SourceStore {
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
Self::write_owned_backup_inner_std(source, base, root, owner, target, bytes, sync_source_root, false, true, false)
|
||||
}
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn write_owned_backup_inner_std(
|
||||
source: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
fresh_owner: bool,
|
||||
sync_parents: bool,
|
||||
source_already_pinned: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
} else {
|
||||
original_path(source, owner)
|
||||
};
|
||||
let _pin = os::pin_multipart_source_directory(source, base, root)?;
|
||||
let _pin = if source_already_pinned {
|
||||
None
|
||||
} else {
|
||||
Some(os::pin_multipart_source_directory(source, base, root)?)
|
||||
};
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("source backup has no parent"))?;
|
||||
let _parents = os::mkdir_all_below_existing_base_std(parent, base, root)?;
|
||||
let existed = if target {
|
||||
let existed = if fresh_owner {
|
||||
false
|
||||
} else if target {
|
||||
match os::observe_multipart_source_metadata(&path, base, root, bytes.len() as u64) {
|
||||
Ok((length, sha)) if length == bytes.len() as u64 && sha == raw_sha(bytes) => true,
|
||||
Ok(_) => {
|
||||
@@ -754,7 +807,10 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
if sync_parents {
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
@@ -1979,25 +2035,28 @@ impl NativeMultipartPublication {
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if cfg!(test) || !can_mark_presynced {
|
||||
let current =
|
||||
load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
@@ -2021,11 +2080,41 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
false,
|
||||
&original_bytes,
|
||||
false,
|
||||
true,
|
||||
old_metadata.is_none(),
|
||||
true,
|
||||
)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
true,
|
||||
previous,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
false,
|
||||
Some(directory_guard),
|
||||
)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -2034,9 +2123,28 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
true,
|
||||
previous,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
false,
|
||||
Some(directory_guard),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
@@ -2050,7 +2158,7 @@ impl NativeMultipartPublication {
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root, Some(directory_guard))
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
@@ -2160,7 +2268,7 @@ impl NativeMultipartPublication {
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
|
||||
prepared_metadata_source.write_all(&new_bytes, true)?;
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root, Some(&owned_directory_guard))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
@@ -2207,7 +2315,7 @@ impl NativeMultipartPublication {
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(self.store.lease.clone(), move || {
|
||||
os::install_multipart_prepared_head(prepared, &canonical, &base, &root)
|
||||
os::install_multipart_prepared_head(prepared, &canonical, &base, &root, None)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -2264,7 +2372,9 @@ impl NativeMultipartPublication {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
fused_in_place: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
directory_guard: Option<&os::RenameCommitGuard>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
@@ -2293,62 +2403,72 @@ impl NativeMultipartPublication {
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
if !fused_in_place {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
})?;
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
!fused_in_place,
|
||||
directory_guard,
|
||||
)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
@@ -2381,7 +2501,7 @@ impl NativeMultipartPublication {
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
this.finish_on_thread(target_already_synced, false, staging_cleanup, None)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -455,8 +455,11 @@ impl LocalDisk {
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
super::multipart_source_state::get_active_multipart_source_part_with_head(
|
||||
&source_object,
|
||||
part.number,
|
||||
from_active_cache,
|
||||
) {
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
|
||||
@@ -2222,6 +2222,36 @@ pub struct WalkDirOptions {
|
||||
// Override the remote stream stall timeout for long background walks.
|
||||
#[serde(default)]
|
||||
pub stall_timeout_ms: Option<u64>,
|
||||
|
||||
/// Select this leaf below identifier-like directory components. The root is
|
||||
/// `base_dir`; unrelated object metadata must not be read or returned.
|
||||
/// Older peers ignore this additive field, so consumers must also filter.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub identifier_tree_leaf: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) fn is_identifier_tree_component(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
|
||||
}
|
||||
|
||||
pub(crate) fn is_identifier_tree_leaf(value: &str) -> bool {
|
||||
value.len() <= 255
|
||||
&& value.contains('.')
|
||||
&& !matches!(value, "." | "..")
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
|
||||
}
|
||||
|
||||
pub(crate) fn identifier_tree_leaf_matches(prefix: &str, leaf: &str, object: &str) -> bool {
|
||||
let Some(relative) = object.strip_prefix(prefix) else {
|
||||
return false;
|
||||
};
|
||||
let mut components = relative.split('/');
|
||||
components.next_back() == Some(leaf) && components.all(is_identifier_tree_component)
|
||||
}
|
||||
|
||||
impl WalkDirOptions {
|
||||
@@ -3157,6 +3187,7 @@ mod tests {
|
||||
skip_total_timeout: false,
|
||||
timeout_ms: Some(10_000),
|
||||
stall_timeout_ms: Some(20_000),
|
||||
identifier_tree_leaf: None,
|
||||
};
|
||||
|
||||
assert_eq!(opts.bucket, "test-bucket");
|
||||
@@ -3191,6 +3222,69 @@ mod tests {
|
||||
assert!(!decoded.skip_hidden_prefix_check);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifier_tree_leaf_selection_preserves_nested_identifier_directories() {
|
||||
let prefix = "catalog/namespaces/";
|
||||
let leaf = "table-entry.json";
|
||||
for relative in [
|
||||
"sales/tables/orders/table-entry.json",
|
||||
"sales/tables/orders/maintenance/tables/owner/table-entry.json",
|
||||
"maintenance/tables/tables/table-entry.json",
|
||||
] {
|
||||
assert!(identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
|
||||
}
|
||||
for relative in [
|
||||
"sales/tables/orders/not-table-entry.json",
|
||||
"sales/tables/orders/table-entry.json/child",
|
||||
"sales/report.json/tables/owner/table-entry.json",
|
||||
"sales//tables/owner/table-entry.json",
|
||||
"sales/../tables/owner/table-entry.json",
|
||||
] {
|
||||
assert!(!identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
|
||||
}
|
||||
assert!(!identifier_tree_leaf_matches(prefix, leaf, "other/sales/tables/orders/table-entry.json"));
|
||||
assert!(is_identifier_tree_leaf(leaf));
|
||||
for invalid in ["", ".", "..", "table-entry", "table/entry.json", "table\\entry.json"] {
|
||||
assert!(!is_identifier_tree_leaf(invalid), "invalid selector {invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifier_tree_walk_options_are_additive_on_json_and_messagepack_maps() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct LegacyView {
|
||||
bucket: String,
|
||||
base_dir: String,
|
||||
recursive: bool,
|
||||
}
|
||||
let options = WalkDirOptions {
|
||||
bucket: "catalog".to_string(),
|
||||
base_dir: "namespaces/".to_string(),
|
||||
recursive: true,
|
||||
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let json = serde_json::to_vec(&options).expect("encode current HTTP options");
|
||||
let messagepack = rmp_serde::to_vec_named(&options).expect("encode legacy RPC map");
|
||||
for old in [
|
||||
serde_json::from_slice::<LegacyView>(&json).expect("old JSON readers ignore the additive field"),
|
||||
rmp_serde::from_slice::<LegacyView>(&messagepack).expect("old MessagePack readers ignore the additive field"),
|
||||
] {
|
||||
assert_eq!(old.bucket, options.bucket);
|
||||
assert_eq!(old.base_dir, options.base_dir);
|
||||
assert!(old.recursive);
|
||||
}
|
||||
let mut old = serde_json::to_value(&options).expect("encode legacy options");
|
||||
old.as_object_mut().expect("options are a map").remove("identifier_tree_leaf");
|
||||
let from_json: WalkDirOptions = serde_json::from_value(old.clone()).expect("decode old HTTP options");
|
||||
let from_messagepack: WalkDirOptions =
|
||||
rmp_serde::from_slice(&rmp_serde::to_vec_named(&old).expect("encode old RPC map")).expect("decode old RPC map");
|
||||
assert!(from_json.identifier_tree_leaf.is_none());
|
||||
assert!(from_messagepack.identifier_tree_leaf.is_none());
|
||||
assert_eq!(from_json.base_dir, options.base_dir);
|
||||
assert_eq!(from_messagepack.base_dir, options.base_dir);
|
||||
}
|
||||
|
||||
/// Test DeleteOptions structure
|
||||
#[test]
|
||||
fn test_delete_options() {
|
||||
|
||||
+104
-15
@@ -611,6 +611,10 @@ pub(crate) mod prepare_rename_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_after_component_opened() -> bool {
|
||||
!AFTER_COMPONENT_OPENED.lock().is_empty()
|
||||
}
|
||||
|
||||
/// Hooks keyed by a directory component name, run before a Windows
|
||||
/// directory pin retries a transient sharing conflict on it.
|
||||
#[cfg(windows)]
|
||||
@@ -2200,6 +2204,7 @@ pub(crate) const MAX_FSYNC_WORKERS_PER_DISK: usize = 8;
|
||||
// Return busy workers to the shared pool so other disks can acquire a slot.
|
||||
const MAX_BLOCKING_JOBS_PER_WORKER: usize = 64;
|
||||
const WORKER_IDLE_SPIN_ITERS: usize = 256;
|
||||
const FSYNC_WORKER_IDLE_SPIN_ITERS: usize = 1024;
|
||||
const FALLBACK_NAMESPACE_QUEUE_SHARDS: usize = 8;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
@@ -2408,7 +2413,7 @@ impl DiskBlockingQueue {
|
||||
.ready
|
||||
.store(!state.jobs.is_empty(), std::sync::atomic::Ordering::Relaxed);
|
||||
if job.is_none() {
|
||||
if state.idle_workers > 0 {
|
||||
if lane != DiskBlockingLane::Fsync && state.idle_workers > 0 {
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
guard.state = DiskBlockingWorkerState::Retired;
|
||||
return;
|
||||
@@ -2426,7 +2431,12 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
idle_polled = false;
|
||||
} else {
|
||||
for _ in 0..WORKER_IDLE_SPIN_ITERS {
|
||||
let spin_iters = if lane == DiskBlockingLane::Fsync {
|
||||
FSYNC_WORKER_IDLE_SPIN_ITERS
|
||||
} else {
|
||||
WORKER_IDLE_SPIN_ITERS
|
||||
};
|
||||
for _ in 0..spin_iters {
|
||||
if queue.lane(lane).ready.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
@@ -3798,7 +3808,7 @@ pub(crate) fn rename_all_with_precondition_and_parent_sync_std(
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run_rename_destination(source, destination);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
{
|
||||
if prepared_publication_test_hooks::has_any_hooks() {
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
|
||||
if let Some(check) = precondition {
|
||||
@@ -7192,6 +7202,39 @@ pub(crate) fn pin_multipart_source_directory(
|
||||
}
|
||||
}
|
||||
|
||||
fn same_parent_commit_guard(
|
||||
parent: &Path,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
directory_guard: Option<&RenameCommitGuard>,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(directory_guard) = directory_guard.filter(|g| g.directories.source_path == parent) {
|
||||
let dir = &directory_guard.directories.source;
|
||||
return Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: directory_guard.directories.root.clone(),
|
||||
source_path: parent.to_path_buf(),
|
||||
destination_path: parent.to_path_buf(),
|
||||
source: LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
},
|
||||
destination: std::sync::OnceLock::from(LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
}),
|
||||
destination_base: base.to_path_buf(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = directory_guard;
|
||||
prepare_rename_commit_guard(parent, parent, base, publication_root)
|
||||
}
|
||||
|
||||
/// A small control record always gets its own inode and an atomic replacement.
|
||||
/// The caller owns the upload SourceRW and keeps its namespace lease alive.
|
||||
pub(crate) fn replace_multipart_source_metadata(
|
||||
@@ -7199,11 +7242,21 @@ pub(crate) fn replace_multipart_source_metadata(
|
||||
bytes: &[u8],
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
replace_multipart_source_metadata_with_guard(path, bytes, base, publication_root, None)
|
||||
}
|
||||
|
||||
pub(crate) fn replace_multipart_source_metadata_with_guard(
|
||||
path: &Path,
|
||||
bytes: &[u8],
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
directory_guard: Option<&RenameCommitGuard>,
|
||||
) -> io::Result<()> {
|
||||
let parent = path.parent().ok_or_else(|| io::Error::other("MPU record has no parent"))?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _pinned = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
|
||||
let guard = same_parent_commit_guard(parent, base, publication_root, directory_guard)?;
|
||||
let staged = parent.join(format!(".mpu-record-{}", uuid::Uuid::new_v4()));
|
||||
let mut prepared = create_prepared_rename_source_with_commit_guard(&staged, path, &guard)?;
|
||||
prepared.write_all(bytes, true)?;
|
||||
@@ -7443,6 +7496,7 @@ pub(crate) fn install_multipart_prepared_head(
|
||||
canonical_head: &Path,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
directory_guard: Option<&RenameCommitGuard>,
|
||||
) -> io::Result<PreparedRenameSource> {
|
||||
let original_path = prepared.path.clone();
|
||||
let parent = canonical_head
|
||||
@@ -7453,7 +7507,7 @@ pub(crate) fn install_multipart_prepared_head(
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _pinned = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
|
||||
let guard = same_parent_commit_guard(parent, base, publication_root, directory_guard)?;
|
||||
rename_prepared_source_std(&prepared, &original_path, canonical_head, base, publication_root, &guard)?;
|
||||
prepared.path = canonical_head.to_path_buf();
|
||||
fsync_commit_directory_limited_std(parent, &guard)?;
|
||||
@@ -7755,8 +7809,7 @@ pub(crate) fn hardlink_multipart_source_files(
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{AtFlags, Mode, OFlags, linkat, openat, unlinkat};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use rustix::fs::{AtFlags, FileType, linkat, statat, unlinkat};
|
||||
#[cfg(test)]
|
||||
let source_path = source.to_path_buf();
|
||||
#[cfg(test)]
|
||||
@@ -7771,20 +7824,26 @@ pub(crate) fn hardlink_multipart_source_files(
|
||||
.ok_or_else(|| io::Error::other("missing multipart attempt directory"))?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_hardlink_destination(&source_path, &destination_path);
|
||||
let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
for name in names {
|
||||
if cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let original = std::fs::File::from(openat(source, name.as_str(), flags, Mode::empty())?);
|
||||
let expected = original.metadata()?;
|
||||
if !expected.is_file() {
|
||||
let expected = statat(source, name.as_str(), AtFlags::SYMLINK_NOFOLLOW)?;
|
||||
let expected_type = FileType::from_raw_mode(expected.st_mode);
|
||||
if expected_type.is_symlink() {
|
||||
return Err(io::Error::from_raw_os_error(libc::ELOOP));
|
||||
}
|
||||
if !expected_type.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart shard is not a regular file"));
|
||||
}
|
||||
linkat(source, name.as_str(), destination, name.as_str(), AtFlags::empty())?;
|
||||
let linked = std::fs::File::from(openat(destination, name.as_str(), flags, Mode::empty())?);
|
||||
let actual = linked.metadata()?;
|
||||
if actual.dev() != expected.dev() || actual.ino() != expected.ino() {
|
||||
let actual = statat(destination, name.as_str(), AtFlags::SYMLINK_NOFOLLOW)?;
|
||||
if !FileType::from_raw_mode(actual.st_mode).is_file()
|
||||
|| actual.st_dev != expected.st_dev
|
||||
|| actual.st_ino != expected.st_ino
|
||||
|| actual.st_size != expected.st_size
|
||||
|| actual.st_mtime != expected.st_mtime
|
||||
{
|
||||
let _ = unlinkat(destination, name.as_str(), AtFlags::empty());
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart shard changed during staging"));
|
||||
}
|
||||
@@ -8205,7 +8264,37 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = publication_root.open_directory(base_dir)?.into();
|
||||
let base = {
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
|
||||
let owned_base = (base_dir != publication_root.io_path.as_path() && base_dir != publication_root.path.as_path())
|
||||
.then(|| publication_root.open_directory(base_dir).map(std::os::fd::OwnedFd::from))
|
||||
.transpose()?;
|
||||
#[cfg(test)]
|
||||
let has_hook = prepare_rename_test_hooks::has_after_component_opened();
|
||||
#[cfg(not(test))]
|
||||
let has_hook = false;
|
||||
if !has_hook {
|
||||
let base_fd = owned_base
|
||||
.as_ref()
|
||||
.map_or_else(|| publication_root.directory.as_fd(), AsFd::as_fd);
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(Errno::NOENT | Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
match owned_base {
|
||||
Some(base) => base,
|
||||
None => publication_root.open_directory(base_dir)?.into(),
|
||||
}
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
// The configured endpoint is trusted; every component beneath it
|
||||
|
||||
@@ -58,7 +58,7 @@ use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
@@ -187,6 +187,38 @@ impl DecommissionCountLedger {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ListReadBudget {
|
||||
permits: Arc<Semaphore>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl ListReadBudget {
|
||||
fn new(capacity: usize) -> Self {
|
||||
let capacity = capacity
|
||||
.min(Semaphore::MAX_PERMITS)
|
||||
.min(usize::try_from(u32::MAX).unwrap_or(usize::MAX));
|
||||
Self {
|
||||
permits: Arc::new(Semaphore::new(capacity)),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire(&self, fanout: usize) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
|
||||
if self.capacity == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
// A single walk must remain admissible even when the topology exceeds
|
||||
// the configured budget. Tokio accepts at most u32::MAX in one grant.
|
||||
let weight = u32::try_from(fanout.max(1).min(self.capacity)).unwrap_or(u32::MAX);
|
||||
self.permits
|
||||
.clone()
|
||||
.acquire_many_owned(weight)
|
||||
.await
|
||||
.map(|permit| Some(Arc::new(permit)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime state owned by a single `ECStore` instance.
|
||||
///
|
||||
/// This is intentionally minimal in the first migration slice; subsequent
|
||||
@@ -201,6 +233,7 @@ pub struct InstanceContext {
|
||||
/// drift out of sync). Stored as one value so the three predicates can
|
||||
/// never observe a torn intermediate state.
|
||||
erasure_kind: RwLock<SetupType>,
|
||||
list_read_budget: OnceLock<ListReadBudget>,
|
||||
/// This instance's namespace lock manager (Phase 5 Slice 3, backlog#939).
|
||||
///
|
||||
/// Owned per-instance so two instances no longer share a lock namespace
|
||||
@@ -354,6 +387,24 @@ impl InstanceContext {
|
||||
Self::with_lock_manager(Arc::new(GlobalLockManager::new()))
|
||||
}
|
||||
|
||||
/// Bound native listing fanout independently of GET body permits. A walk
|
||||
/// keeps its shared grant until all of its disk producers have terminated.
|
||||
/// Acquire before creating the merge channels or taking object locks.
|
||||
pub(crate) async fn acquire_list_read_budget(
|
||||
&self,
|
||||
fanout: usize,
|
||||
) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
|
||||
self.list_read_budget
|
||||
.get_or_init(|| {
|
||||
ListReadBudget::new(rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
))
|
||||
})
|
||||
.acquire(fanout)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Build a context bound to a specific lock manager.
|
||||
///
|
||||
/// [`bootstrap_ctx`] uses this to alias the process-global lock manager so
|
||||
@@ -362,6 +413,7 @@ impl InstanceContext {
|
||||
fn with_lock_manager(lock_manager: Arc<GlobalLockManager>) -> Self {
|
||||
Self {
|
||||
erasure_kind: RwLock::new(SetupType::Unknown),
|
||||
list_read_budget: OnceLock::new(),
|
||||
lock_manager,
|
||||
region: OnceLock::new(),
|
||||
deployment_id: OnceLock::new(),
|
||||
@@ -1111,6 +1163,85 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_weights_and_clamps_fanout() {
|
||||
let budget = ListReadBudget::new(8);
|
||||
let first = budget.acquire(3).await.expect("first walk").expect("bounded grant");
|
||||
assert_eq!(budget.permits.available_permits(), 5);
|
||||
let mut blocked = std::pin::pin!(budget.acquire(6));
|
||||
assert!(futures::poll!(&mut blocked).is_pending());
|
||||
drop(first);
|
||||
let second = blocked.await.expect("queued walk").expect("bounded grant");
|
||||
assert_eq!(budget.permits.available_permits(), 2);
|
||||
drop(second);
|
||||
let oversized = budget
|
||||
.acquire(usize::MAX)
|
||||
.await
|
||||
.expect("oversized walk")
|
||||
.expect("bounded grant");
|
||||
assert_eq!(budget.permits.available_permits(), 0);
|
||||
drop(oversized);
|
||||
assert_eq!(budget.permits.available_permits(), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_zero_preserves_unbounded_configuration() {
|
||||
let budget = ListReadBudget::new(0);
|
||||
assert!(budget.acquire(usize::MAX).await.expect("disabled budget").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_cancelled_waiter_returns_partial_grant() {
|
||||
let budget = ListReadBudget::new(4);
|
||||
let held = budget.acquire(3).await.expect("first walk");
|
||||
{
|
||||
let mut waiter = std::pin::pin!(budget.acquire(4));
|
||||
assert!(futures::poll!(&mut waiter).is_pending());
|
||||
}
|
||||
assert_eq!(budget.permits.available_permits(), 1);
|
||||
drop(held);
|
||||
assert!(budget.acquire(4).await.expect("next walk").is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_shared_grant_survives_parent_drop() {
|
||||
let budget = ListReadBudget::new(4);
|
||||
let parent = budget.acquire(4).await.expect("first walk").expect("bounded grant");
|
||||
let producer = parent.clone();
|
||||
drop(parent);
|
||||
assert_eq!(budget.permits.available_permits(), 0);
|
||||
drop(producer);
|
||||
assert_eq!(budget.permits.available_permits(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_is_shared_by_context_and_isolated_between_instances() {
|
||||
let first = Arc::new(InstanceContext::new());
|
||||
first.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
|
||||
let same = first.clone();
|
||||
let other = InstanceContext::new();
|
||||
other.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
|
||||
let held = first.acquire_list_read_budget(4).await.expect("first context");
|
||||
let mut waiter = std::pin::pin!(same.acquire_list_read_budget(1));
|
||||
assert!(futures::poll!(&mut waiter).is_pending());
|
||||
assert!(
|
||||
other
|
||||
.acquire_list_read_budget(4)
|
||||
.await
|
||||
.expect("independent context")
|
||||
.is_some()
|
||||
);
|
||||
drop(held);
|
||||
assert!(waiter.await.expect("shared context resumed").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_read_budget_extreme_configuration_is_bounded_without_panicking() {
|
||||
let budget = ListReadBudget::new(usize::MAX);
|
||||
assert!(budget.capacity <= Semaphore::MAX_PERMITS);
|
||||
assert!(u32::try_from(budget.capacity).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_count_ledger_bounds_retained_entries() {
|
||||
let mut ledger = super::DecommissionCountLedger::default();
|
||||
|
||||
@@ -593,7 +593,7 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||
pool_meta_guard.ensure_write_safe(stage)?;
|
||||
let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
|
||||
let pool_meta = self
|
||||
@@ -625,7 +625,7 @@ impl ECStore {
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
||||
// shared fences admit a peer without waiting for the run's own entries.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
||||
let pool_meta = self
|
||||
@@ -654,7 +654,7 @@ impl ECStore {
|
||||
{
|
||||
// The admission read fences must already be dropped. Reacquire in the
|
||||
// original order and revalidate every durable boundary across that gap.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let pool_meta = self
|
||||
@@ -1714,7 +1714,7 @@ mod tests {
|
||||
persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
|
||||
.await
|
||||
.expect("initialized pool metadata identity should persist");
|
||||
*store.pool_meta_save_gate.lock().await = write_state;
|
||||
*store.pool_meta_save_gate.write().await = write_state;
|
||||
// Simulate physical loss; raw deletion of authoritative pool.bin is
|
||||
// deliberately rejected by the native metadata mutation guard.
|
||||
for pool in &store.pools {
|
||||
|
||||
@@ -100,7 +100,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
||||
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
@@ -335,7 +335,7 @@ async fn test_pool_stores_with_contexts(
|
||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
ctx: store_ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
})
|
||||
|
||||
@@ -3412,7 +3412,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
||||
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
})
|
||||
|
||||
@@ -93,6 +93,7 @@ use super::{
|
||||
delete_tier_mutation_intent_record_if_current, list_tier_coordinator_mutation_intent_records,
|
||||
list_tier_mutation_intent_records, load_tier_coordinator_mutation_intent_record_with_etag,
|
||||
load_tier_mutation_intent_record_with_etag, save_tier_coordinator_mutation_intent_record_if_absent,
|
||||
save_tier_mutation_intent_record_if_current,
|
||||
},
|
||||
warm_backend::WarmBackendImpl,
|
||||
};
|
||||
@@ -620,7 +621,11 @@ struct CommittedMutationRecoveryPlan {
|
||||
|
||||
impl TierMutationRecoverySnapshot {
|
||||
fn requires_config_lock(&self) -> bool {
|
||||
self.has_committed_mutation_blocks || self.intents.iter().any(|recovered| recovered.has_coordinator_record)
|
||||
self.has_committed_mutation_blocks
|
||||
|| self.intents.iter().any(|recovered| {
|
||||
recovered.has_coordinator_record
|
||||
|| (recovered.has_peer_record && recovered.intent.state == TierMutationIntentState::Prepared)
|
||||
})
|
||||
}
|
||||
|
||||
fn allowance(&self) -> MutationBlockAllowance {
|
||||
@@ -5102,6 +5107,9 @@ impl TierConfigMgr {
|
||||
|
||||
Self::reconcile_terminal_dual_prefix_mutation_intents(api.clone(), &mut snapshot).await?;
|
||||
Self::recover_prepared_coordinator_mutation_intents(api.clone(), &mut snapshot.coordinator_intents).await?;
|
||||
if config_lock.is_some() {
|
||||
Self::recover_expired_peer_only_prepares(api.clone(), &snapshot).await?;
|
||||
}
|
||||
|
||||
snapshot = Self::load_and_reconcile_mutation_recovery_snapshot(handle, api.clone(), true).await?;
|
||||
if snapshot.requires_config_lock() && config_lock.is_none() {
|
||||
@@ -5502,6 +5510,58 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_expired_peer_only_prepares<S>(api: Arc<S>, snapshot: &TierMutationRecoverySnapshot) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
|
||||
let orphaned = snapshot
|
||||
.intents
|
||||
.iter()
|
||||
.filter(|recovered| {
|
||||
recovered.has_peer_record
|
||||
&& !recovered.has_coordinator_record
|
||||
&& recovered.intent.state == TierMutationIntentState::Prepared
|
||||
&& Self::tier_mutation_tombstone_retention_elapsed(&recovered.intent, now)
|
||||
})
|
||||
.map(|recovered| recovered.intent.clone())
|
||||
.collect::<Vec<_>>();
|
||||
if orphaned.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// The caller holds the tier-config namespace lock. Only the exact
|
||||
// persisted candidate proves a commit; an old or different config
|
||||
// leaves the peer fence intact.
|
||||
let loaded = load_tier_config_for_recovery(api.clone()).await?;
|
||||
let Some(config_etag) = loaded.etag else {
|
||||
return Ok(());
|
||||
};
|
||||
for intent in orphaned {
|
||||
if loaded.persisted_digest != Some(intent.candidate_digest) {
|
||||
continue;
|
||||
}
|
||||
let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await;
|
||||
let (mut current, record_etag) =
|
||||
match load_tier_mutation_intent_record_with_etag(api.clone(), intent.mutation_id).await {
|
||||
Ok(record) => record,
|
||||
Err(Error::ConfigNotFound) => continue,
|
||||
Err(err) => return Err(tier_mutation_replay_error(err)),
|
||||
};
|
||||
if current != intent {
|
||||
continue;
|
||||
}
|
||||
current
|
||||
.advance(TierMutationIntentState::Committed, Some(config_etag.clone()))
|
||||
.map_err(io::Error::other)?;
|
||||
match save_tier_mutation_intent_record_if_current(api.clone(), ¤t, &record_etag).await {
|
||||
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||
Err(err) => return Err(tier_mutation_replay_error(err)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recover_prepared_coordinator_mutation_intents<S>(api: Arc<S>, intents: &mut [TierMutationIntent]) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
@@ -13379,6 +13439,208 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_recovers_expired_peer_only_prepare_for_persisted_add() {
|
||||
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let previous = empty_mgr();
|
||||
let mut candidate = empty_mgr();
|
||||
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
let targets = build_tier_mutation_affected_targets(
|
||||
TierMutationIntentKind::Add,
|
||||
HashSet::from(["COLD-A".to_string()]),
|
||||
&previous,
|
||||
&candidate,
|
||||
)
|
||||
.expect("added tier should have a target");
|
||||
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||
.expect("intent should build")
|
||||
.expect("add should require an intent");
|
||||
intent.expires_at_unix_nanos = 1;
|
||||
candidate
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("candidate should persist");
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("peer prepare should persist without a coordinator record");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(Vec::new(), async {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(healthy_driver_factory(), async {
|
||||
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||
.await
|
||||
.expect("matching saved candidate should recover");
|
||||
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("recovered tier should be readable");
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
TierConfigMgr::load_tier_mutation_intents(store)
|
||||
.await
|
||||
.expect("peer scan should succeed")
|
||||
.iter()
|
||||
.all(|record| record.state != TierMutationIntentState::Prepared),
|
||||
"the orphaned prepare must no longer block later reloads"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_keeps_peer_only_prepare_without_commit_proof() {
|
||||
use crate::services::tier::tier_mutation_intent::{
|
||||
save_tier_mutation_intent_record, tier_mutation_intent_record_object_name,
|
||||
};
|
||||
|
||||
for case in [
|
||||
"unexpired",
|
||||
"recently_expired",
|
||||
"different_candidate",
|
||||
"missing_config",
|
||||
"concurrent_change",
|
||||
] {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let previous = empty_mgr();
|
||||
let mut candidate = empty_mgr();
|
||||
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
let targets = build_tier_mutation_affected_targets(
|
||||
TierMutationIntentKind::Add,
|
||||
HashSet::from(["COLD-A".to_string()]),
|
||||
&previous,
|
||||
&candidate,
|
||||
)
|
||||
.expect("added tier should have a target");
|
||||
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||
.expect("intent should build")
|
||||
.expect("add should require an intent");
|
||||
intent.expires_at_unix_nanos = match case {
|
||||
"unexpired" => tier_mutation_intent_expiry_unix_nanos(),
|
||||
"recently_expired" => {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).expect("test clock should fit")
|
||||
- 1_000_000_000
|
||||
}
|
||||
_ => 1,
|
||||
};
|
||||
if case == "different_candidate" {
|
||||
intent.candidate_digest = [0x51; 32];
|
||||
}
|
||||
if case != "missing_config" {
|
||||
candidate
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("candidate should persist");
|
||||
}
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("peer prepare should persist");
|
||||
let expected = if case == "concurrent_change" {
|
||||
let mut changed = intent.clone();
|
||||
changed.candidate_digest = [0x52; 32];
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).expect("record path should build");
|
||||
store
|
||||
.rewrite_on_next_if_match(object, changed.encode().expect("changed intent should encode"))
|
||||
.await;
|
||||
changed
|
||||
} else {
|
||||
intent
|
||||
};
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(Vec::new(), async {
|
||||
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||
.await
|
||||
.expect("unproven prepare should remain fenced");
|
||||
})
|
||||
.await;
|
||||
let error = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("{case}: unproven prepare must block reads"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.message.contains("being replaced"), "{case}: {error}");
|
||||
let records = TierConfigMgr::load_tier_mutation_intents(store)
|
||||
.await
|
||||
.expect("peer scan should succeed");
|
||||
assert_eq!(records, vec![expected], "{case}: prepared evidence must remain unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_only_prepare_recovery_retains_fence_until_peer_commit_succeeds() {
|
||||
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let previous = empty_mgr();
|
||||
let mut candidate = empty_mgr();
|
||||
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
let targets = build_tier_mutation_affected_targets(
|
||||
TierMutationIntentKind::Add,
|
||||
HashSet::from(["COLD-A".to_string()]),
|
||||
&previous,
|
||||
&candidate,
|
||||
)
|
||||
.expect("added tier should have a target");
|
||||
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||
.expect("intent should build")
|
||||
.expect("add should require an intent");
|
||||
intent.expires_at_unix_nanos = 1;
|
||||
candidate
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("candidate should persist");
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("peer prepare should persist");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(vec![FakeTierMutationPeer::boxed("peer-a", calls.clone(), Err("offline"))], async {
|
||||
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||
.await
|
||||
.expect_err("failed peer commit must leave recovery incomplete");
|
||||
})
|
||||
.await;
|
||||
let retained = TierConfigMgr::load_tier_mutation_intents(store.clone())
|
||||
.await
|
||||
.expect("peer scan should succeed");
|
||||
assert_eq!(retained.len(), 1);
|
||||
assert_eq!(retained[0].state, TierMutationIntentState::Committed);
|
||||
let error = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("failed peer commit must retain the fence"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.message.contains("being replaced"), "{error}");
|
||||
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(
|
||||
vec![FakeTierMutationPeer::boxed(
|
||||
"peer-a",
|
||||
calls.clone(),
|
||||
Ok(PeerTierMutationState::Committed),
|
||||
)],
|
||||
async {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(healthy_driver_factory(), async {
|
||||
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||
.await
|
||||
.expect("peer commit retry should recover");
|
||||
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("tier should be readable after the retry");
|
||||
})
|
||||
.await;
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(lock_unpoisoned(&calls).len(), 2, "recovery should retry the peer commit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_mutation_recovery_scan_blocks_new_operation_lease() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
|
||||
@@ -4608,6 +4608,19 @@ fn marker_purge_receipt_path(
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// The first disk slot UUID is assigned when this erasure set is formatted.
|
||||
/// It survives pool reordering and replacement of a disk in that slot.
|
||||
pub fn persistent_set_id(&self) -> Result<Uuid> {
|
||||
self.format
|
||||
.erasure
|
||||
.sets
|
||||
.get(self.set_index)
|
||||
.and_then(|disks| disks.first())
|
||||
.copied()
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("erasure set has no persistent format identity"))
|
||||
}
|
||||
|
||||
/// Persist a quorum DELETE receipt outside the object marker itself. The
|
||||
/// receipt is written to the metadata bucket so it survives a node restart
|
||||
/// while an inaccessible member still holds the stale marker.
|
||||
|
||||
@@ -2979,10 +2979,11 @@ fn build_legacy_per_part_fallback_reader(
|
||||
part_length: usize,
|
||||
part_size: usize,
|
||||
read_repair: Option<(DecodeReadRepairContext, bool)>,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
|
||||
) -> Box<LegacyPerPartDecodeReader> {
|
||||
let buffer = adaptive_duplex_buffer_size(part_size as i64);
|
||||
let (read_half, mut write_half) = tokio::io::duplex(buffer);
|
||||
let (read_half, write_half) = tokio::io::duplex(buffer);
|
||||
let decode = tokio::spawn(async move {
|
||||
let mut write_half = GetObjectDownstreamWriter::new(write_half);
|
||||
let outcome = erasure
|
||||
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
|
||||
&mut write_half,
|
||||
@@ -8908,6 +8909,42 @@ mod tests {
|
||||
assert!(err.to_string().contains("inconsistent read source shards"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_per_part_output_close_retains_downstream_classification() {
|
||||
let erasure = coding::Erasure::new(2, 2, 64);
|
||||
let data = (0..64u8).collect::<Vec<_>>();
|
||||
let setup =
|
||||
setup_codec_data_blocks_first_encoded_bitrot_readers(&erasure, &data, &[0], &[], &[], HashAlgorithm::HighwayHash256)
|
||||
.await;
|
||||
let mut reader = build_legacy_per_part_fallback_reader(
|
||||
erasure,
|
||||
setup.readers,
|
||||
setup.deferred_stripe_handles,
|
||||
Vec::new(),
|
||||
None,
|
||||
0,
|
||||
data.len(),
|
||||
data.len(),
|
||||
None,
|
||||
);
|
||||
// Closing the actual output pipe models the body consumer disappearing
|
||||
// while retaining the executor long enough to inspect its typed result.
|
||||
assert!(!reader.finished);
|
||||
let (replacement, _unused) = tokio::io::duplex(1);
|
||||
drop(std::mem::replace(&mut reader.inner, replacement));
|
||||
let handle = reader.decode.take().expect("fallback decode task");
|
||||
let error = handle
|
||||
.await
|
||||
.expect("join the fallback task")
|
||||
.expect("closed pipe must interrupt output");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe);
|
||||
assert_eq!(
|
||||
crate::diagnostics::get::classify_io_error(&error),
|
||||
GetObjectFailureReason::DownstreamClosed,
|
||||
"fallback output must carry the same marker as the normal GET path"
|
||||
);
|
||||
}
|
||||
|
||||
// backlog#879: a later multipart part that loses a shard (read quorum still
|
||||
// holds) must be reconstructed in place through the legacy per-part decode
|
||||
// bridge instead of failing the stream. This exercises the exact reader the
|
||||
|
||||
@@ -443,6 +443,7 @@ impl ECStore {
|
||||
},
|
||||
other => StorageError::Lock(other),
|
||||
})?;
|
||||
let ns_guard = Arc::new(ns_guard);
|
||||
|
||||
let recover = Box::pin(async {
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket).await?;
|
||||
@@ -930,15 +931,16 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
|
||||
run_bucket_usage_cleanup(guard, bucket, async {
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
|
||||
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) -> Result<()> {
|
||||
let namespace_guard = guard.map(Arc::as_ref);
|
||||
run_bucket_usage_cleanup(namespace_guard, bucket, async {
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, namespace_guard).await?;
|
||||
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
|
||||
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) {
|
||||
if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
@@ -1029,6 +1031,7 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = ns_guard.map(Arc::new);
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
@@ -1146,7 +1149,7 @@ impl ECStore {
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
@@ -1162,7 +1165,7 @@ impl ECStore {
|
||||
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
@@ -1191,9 +1194,9 @@ impl ECStore {
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_deref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_deref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
@@ -1214,10 +1217,10 @@ impl ECStore {
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
|| ns_guard.as_deref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_deref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
@@ -1580,6 +1583,7 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = ns_guard.map(Arc::new);
|
||||
|
||||
let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
|
||||
let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
|
||||
@@ -1615,7 +1619,7 @@ impl ECStore {
|
||||
crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
Some(ns_guard.as_ref().unwrap_or(&guard)),
|
||||
Some(ns_guard.as_deref().unwrap_or(&guard)),
|
||||
)
|
||||
.await?;
|
||||
match state {
|
||||
@@ -1641,7 +1645,7 @@ impl ECStore {
|
||||
if sr_mark_delete {
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
bucket,
|
||||
"bucket delete marker creation",
|
||||
await_bucket_namespace_operation(
|
||||
@@ -1680,7 +1684,7 @@ impl ECStore {
|
||||
object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
|
||||
bucket,
|
||||
"Object Lock configuration fence during bucket deletion",
|
||||
run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
|
||||
run_physical_bucket_deletion(ns_guard.as_deref(), bucket, async {
|
||||
self.delete_bucket_on_sets(bucket, &delete_opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
@@ -1709,12 +1713,12 @@ impl ECStore {
|
||||
if let Some(guard) = bucket_lifecycle_guard.as_ref() {
|
||||
record_opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
if let Some(guard) = ns_guard.as_ref() {
|
||||
if let Some(guard) = ns_guard.as_deref() {
|
||||
record_opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
bucket,
|
||||
"bucket retirement publication",
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
|
||||
@@ -1724,7 +1728,7 @@ impl ECStore {
|
||||
// V1 ledger must not block quota admission after same-name recreation.
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
bucket,
|
||||
"retired bucket quota ledger cleanup",
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
|
||||
@@ -1735,7 +1739,7 @@ impl ECStore {
|
||||
self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
|
||||
|
||||
if let Err(err) = self
|
||||
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
|
||||
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_deref())
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
|
||||
@@ -527,7 +527,7 @@ impl ECStore {
|
||||
async fn acquire_heal_format_fence(
|
||||
&self,
|
||||
) -> Result<(
|
||||
tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
|
||||
crate::core::pools::PoolMetaSaveGateWriteGuard<'_>,
|
||||
NamespaceLockGuard,
|
||||
NamespaceLockGuard,
|
||||
PoolMeta,
|
||||
@@ -538,7 +538,7 @@ impl ECStore {
|
||||
.first()
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
|
||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
||||
let mut write_state = self.pool_meta_save_gate.write().await;
|
||||
write_state.ensure_write_safe("heal format fence failed")?;
|
||||
|
||||
// Metadata fence order is part of the decommission/rebalance protocol:
|
||||
@@ -1079,7 +1079,7 @@ impl ECStore {
|
||||
pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||
let _movement_guard = movement_gate.read().await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let save_guard = self.pool_meta_save_gate.write().await;
|
||||
save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
|
||||
let mut fenced_pools = BTreeSet::new();
|
||||
{
|
||||
@@ -2527,7 +2527,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
@@ -3550,10 +3550,14 @@ mod tests {
|
||||
async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
|
||||
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
||||
needs_repair: true,
|
||||
repair_write_safe: false,
|
||||
});
|
||||
store
|
||||
.pool_meta_save_gate
|
||||
.write()
|
||||
.await
|
||||
.observe_replicas(PoolMetaReplicaState {
|
||||
needs_repair: true,
|
||||
repair_write_safe: false,
|
||||
});
|
||||
|
||||
let err = store
|
||||
.handle_heal_format(false)
|
||||
@@ -3688,7 +3692,7 @@ mod tests {
|
||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
||||
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
|
||||
let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
|
||||
let recovery_latch = store.pool_meta_save_gate.write().await.aborted_transaction_latch_for_test();
|
||||
let mut heal = tokio::spawn({
|
||||
let store = Arc::clone(&store);
|
||||
async move { store.handle_heal_format(false).await }
|
||||
@@ -3848,7 +3852,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
||||
let state = source.pool_meta_save_gate.write().await.independent_clone_for_test();
|
||||
Arc::new(ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: Default::default(),
|
||||
@@ -3858,7 +3862,7 @@ mod tests {
|
||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(state),
|
||||
ctx: Arc::clone(&source.ctx),
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
@@ -4040,7 +4044,7 @@ mod tests {
|
||||
let missing = remove_pool_meta_shard(&store, 0).await;
|
||||
let (guard, admissions, mut state) = store.acquire_pool_meta_object_heal_fence(&[0]).await.expect("heal authority");
|
||||
assert!(admissions.into_iter().all(|admission| admission.is_ok()));
|
||||
assert!(store.pool_meta_save_gate.try_lock().is_err(), "heal must retain the earlier save gate");
|
||||
assert!(store.pool_meta_save_gate.try_write().is_err(), "heal must retain the earlier save gate");
|
||||
let writer_store = Arc::clone(&store);
|
||||
let writer = tokio::spawn(async move { writer_store.save_current_pool_meta_for_test(&[0]).await });
|
||||
tokio::task::yield_now().await;
|
||||
@@ -4687,10 +4691,14 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
|
||||
let store = minimal_heal_store().await;
|
||||
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
||||
needs_repair: true,
|
||||
repair_write_safe: false,
|
||||
});
|
||||
store
|
||||
.pool_meta_save_gate
|
||||
.write()
|
||||
.await
|
||||
.observe_replicas(PoolMetaReplicaState {
|
||||
needs_repair: true,
|
||||
repair_write_safe: false,
|
||||
});
|
||||
|
||||
let err = store
|
||||
.handle_heal_bucket("bucket", &HealOpts::default())
|
||||
@@ -4724,7 +4732,7 @@ mod tests {
|
||||
.await
|
||||
.expect("peer dispatch should start");
|
||||
assert!(
|
||||
store.pool_meta_save_gate.try_lock().is_ok(),
|
||||
store.pool_meta_save_gate.try_write().is_ok(),
|
||||
"coordinator must release its local save gate before waiting for peers"
|
||||
);
|
||||
assert!(
|
||||
@@ -5116,7 +5124,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
};
|
||||
@@ -5136,7 +5144,7 @@ mod tests {
|
||||
.save_for_startup(store.pools.clone())
|
||||
.await
|
||||
.expect("pool metadata should be persisted before format heal");
|
||||
*store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
|
||||
*store.pool_meta_save_gate.write().await = crate::core::pools::PoolMetaWriteState::default();
|
||||
|
||||
let replacement_endpoint = store.pools[1].endpoints.endpoints.as_ref()[2].clone();
|
||||
tokio::fs::remove_file(
|
||||
|
||||
@@ -676,7 +676,7 @@ impl ECStore {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers,
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(pool_meta_write_state),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(pool_meta_write_state),
|
||||
// Adopt the caller's context (the process bootstrap one on the
|
||||
// legacy path) so startup writes (erasure type recorded before
|
||||
// this point) and later reads share one cell.
|
||||
@@ -725,7 +725,7 @@ impl ECStore {
|
||||
// Lock order is save gate -> distributed metadata writer, matching
|
||||
// runtime mutations. RPC locks are available before ECStore publication
|
||||
// and do not depend on the later fleet proof.
|
||||
let mut startup_write_state = self.pool_meta_save_gate.lock().await;
|
||||
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||
let pool = self
|
||||
.pools
|
||||
|
||||
@@ -127,28 +127,30 @@ impl AuthoritativeListOperations for ECStore {
|
||||
marker: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<ListObjectVersionsInfo> {
|
||||
if max_keys <= 0 || max_keys > MAX_OBJECT_LIST {
|
||||
self.list_unversioned_authoritative_page(bucket, prefix, marker, max_keys, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_identifier_tree_leaves_authoritative(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
leaf: &str,
|
||||
marker: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<ListObjectVersionsInfo> {
|
||||
let valid_prefix = prefix
|
||||
.strip_suffix('/')
|
||||
.is_some_and(|root| !root.is_empty() && root.split('/').all(|part| !part.is_empty() && !matches!(part, "." | "..")));
|
||||
if !valid_prefix || !crate::disk::is_identifier_tree_leaf(leaf) {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_string(),
|
||||
prefix.to_string(),
|
||||
"authoritative page size is out of range".to_string(),
|
||||
"identifier-tree listing requires a directory prefix and a dotted leaf name".to_string(),
|
||||
));
|
||||
}
|
||||
let opts = ListPathOptions {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: prefix.to_string(),
|
||||
marker: marker.clone(),
|
||||
limit: max_keys,
|
||||
recursive: true,
|
||||
incl_deleted: true,
|
||||
versioned: true,
|
||||
authoritative: true,
|
||||
caller: ListObjectsGatherCaller::S3ListVersions,
|
||||
..Default::default()
|
||||
};
|
||||
let mut page = self.list_path(&opts).await?;
|
||||
page.entries.forward_past(marker);
|
||||
unversioned_authoritative_page(bucket, page)
|
||||
self.list_unversioned_authoritative_page(bucket, prefix, marker, max_keys, Some(leaf.to_string()))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn walk_authoritative(
|
||||
@@ -368,6 +370,9 @@ fn log_list_path_finished(
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ListPathOptions {
|
||||
pub caller: ListObjectsGatherCaller,
|
||||
/// Internal native-walk admission, shared with every disk producer so
|
||||
/// aborting an HTTP request cannot release its budget ahead of cleanup.
|
||||
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
|
||||
pub id: Option<String>,
|
||||
|
||||
// Bucket of the listing.
|
||||
@@ -433,6 +438,8 @@ pub struct ListPathOptions {
|
||||
// Internal bounded pages require GET-equivalent per-set resolution.
|
||||
pub authoritative: bool,
|
||||
|
||||
pub identifier_tree_leaf: Option<String>,
|
||||
|
||||
pub stop_disk_at_limit: bool,
|
||||
|
||||
pub pool_idx: Option<usize>,
|
||||
@@ -1098,6 +1105,39 @@ impl ListPathOptions {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
async fn list_unversioned_authoritative_page(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
marker: Option<String>,
|
||||
max_keys: i32,
|
||||
identifier_tree_leaf: Option<String>,
|
||||
) -> Result<ListObjectVersionsInfo> {
|
||||
if max_keys <= 0 || max_keys > MAX_OBJECT_LIST {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_string(),
|
||||
prefix.to_string(),
|
||||
"authoritative page size is out of range".to_string(),
|
||||
));
|
||||
}
|
||||
let opts = ListPathOptions {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: prefix.to_string(),
|
||||
marker: marker.clone(),
|
||||
limit: max_keys,
|
||||
recursive: true,
|
||||
incl_deleted: true,
|
||||
versioned: true,
|
||||
authoritative: true,
|
||||
identifier_tree_leaf,
|
||||
caller: ListObjectsGatherCaller::S3ListVersions,
|
||||
..Default::default()
|
||||
};
|
||||
let mut page = self.list_path(&opts).await?;
|
||||
page.entries.forward_past(marker);
|
||||
unversioned_authoritative_page(bucket, page)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
// @continuation_token marker
|
||||
// @start_after as marker when continuation_token empty
|
||||
@@ -1478,6 +1518,11 @@ impl ECStore {
|
||||
if o.transient {
|
||||
o.create = false;
|
||||
}
|
||||
let fanout = self.pools.iter().fold(0usize, |total, pool| {
|
||||
total.saturating_add(pool.disk_set.len().saturating_mul(pool.set_drive_count))
|
||||
});
|
||||
o.read_budget = self.ctx.acquire_list_read_budget(fanout).await.map_err(Error::other)?;
|
||||
|
||||
let log_context = ListPathLogContext::from_options(&o);
|
||||
|
||||
// cancel channel
|
||||
@@ -2878,6 +2923,12 @@ impl SetDisks {
|
||||
if o.transient {
|
||||
o.create = false;
|
||||
}
|
||||
o.read_budget = self
|
||||
.instance_ctx()
|
||||
.acquire_list_read_budget(self.set_drive_count)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
let log_context = ListPathLogContext::from_options(&o);
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
@@ -2922,6 +2973,33 @@ impl SetDisks {
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
fn resolve_authoritative_listed_entry(
|
||||
bucket: &str,
|
||||
selection: Option<(&str, &str)>,
|
||||
entries: MetaCacheEntries,
|
||||
errors: &[Option<DiskError>],
|
||||
disk_count: usize,
|
||||
parity: usize,
|
||||
) -> crate::disk::error::Result<Option<MetaCacheEntry>> {
|
||||
if disk_count == 0 || entries.0.len() > disk_count || entries.0.len() != errors.len() {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
// An old peer may ignore the optional disk selector. Its noncandidate
|
||||
// rows must be ignored before metadata parsing. Selected keys retain
|
||||
// the original resolver and all of its quorum/corruption checks.
|
||||
if selection.is_some_and(|(prefix, leaf)| {
|
||||
entries
|
||||
.0
|
||||
.iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.is_some_and(|entry| !crate::disk::identifier_tree_leaf_matches(prefix, leaf, &entry.name))
|
||||
}) {
|
||||
return Ok(None);
|
||||
}
|
||||
Self::resolve_listed_versions(bucket, entries, errors, disk_count, parity)
|
||||
}
|
||||
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
@@ -2945,10 +3023,13 @@ impl SetDisks {
|
||||
let first_error = Arc::new(tokio::sync::Mutex::new(None));
|
||||
let error_sink = Arc::clone(&first_error);
|
||||
let bucket = opts.bucket.clone();
|
||||
let prefix = opts.prefix.clone();
|
||||
let identifier_tree_leaf = opts.identifier_tree_leaf.clone();
|
||||
let cancel = rx.clone();
|
||||
let result = list_path_raw_with_claim_tracker(
|
||||
rx,
|
||||
ListPathRawOptions {
|
||||
read_budget: opts.read_budget.clone(),
|
||||
disks: disks.into_iter().map(Some).collect(),
|
||||
bucket: opts.bucket,
|
||||
path: opts.base_dir,
|
||||
@@ -2961,6 +3042,7 @@ impl SetDisks {
|
||||
forward_to: opts.marker,
|
||||
min_disks: read_quorum,
|
||||
preserve_replica_metadata: true,
|
||||
identifier_tree_leaf: opts.identifier_tree_leaf,
|
||||
// A reader's local page may be consumed by stale entries. Only
|
||||
// the resolved, merged page may stop the authoritative walk.
|
||||
per_disk_limit: 0,
|
||||
@@ -2968,7 +3050,14 @@ impl SetDisks {
|
||||
walkdir_timeout: opts.walkdir_timeout,
|
||||
walkdir_stall_timeout: opts.walkdir_stall_timeout,
|
||||
partial: Some(Box::new(move |entries, errors| {
|
||||
let resolved = Self::resolve_listed_versions(&bucket, entries, errors, disk_count, parity);
|
||||
let resolved = Self::resolve_authoritative_listed_entry(
|
||||
&bucket,
|
||||
identifier_tree_leaf.as_deref().map(|leaf| (prefix.as_str(), leaf)),
|
||||
entries,
|
||||
errors,
|
||||
disk_count,
|
||||
parity,
|
||||
);
|
||||
let sender = sender.clone();
|
||||
let cancel = cancel.clone();
|
||||
let first_error = Arc::clone(&error_sink);
|
||||
@@ -3107,6 +3196,7 @@ impl SetDisks {
|
||||
let result = list_path_raw_with_outcome(
|
||||
rx,
|
||||
ListPathRawOptions {
|
||||
read_budget: opts.read_budget.clone(),
|
||||
disks: disks.into_iter().map(Some).collect(),
|
||||
fallback_disks: fallback_disks.into_iter().map(Some).collect(),
|
||||
bucket: opts.bucket,
|
||||
@@ -6788,6 +6878,58 @@ mod test {
|
||||
(dirs, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_read_budget_bounds_store_and_set_entrypoints_and_recovers_after_errors() {
|
||||
let (_dirs, store) = single_set_store(4).await;
|
||||
let held = store
|
||||
.ctx
|
||||
.acquire_list_read_budget(usize::MAX)
|
||||
.await
|
||||
.expect("occupy native budget");
|
||||
let semaphore = held.as_ref().expect("bounded instance budget").semaphore();
|
||||
let owners = Arc::strong_count(semaphore);
|
||||
let opts = ListPathOptions {
|
||||
bucket: "list-budget-absent-bucket".into(),
|
||||
limit: 1,
|
||||
authoritative: true,
|
||||
..Default::default()
|
||||
};
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
set.disks.write().await.fill(None);
|
||||
{
|
||||
let mut store_walk = std::pin::pin!(store.clone().list_path(&opts));
|
||||
let mut set_walk = std::pin::pin!(set.clone().list_path_result(&opts));
|
||||
assert!(futures::poll!(&mut store_walk).is_pending(), "store walk must await its budget");
|
||||
assert_eq!(Arc::strong_count(semaphore), owners + 1, "store must queue on the shared budget");
|
||||
assert!(futures::poll!(&mut set_walk).is_pending(), "set walk shares the same instance budget");
|
||||
assert_eq!(Arc::strong_count(semaphore), owners + 2, "set must queue on the same budget");
|
||||
let empty = store
|
||||
.clone()
|
||||
.list_path(&ListPathOptions {
|
||||
limit: 0,
|
||||
..opts.clone()
|
||||
})
|
||||
.await
|
||||
.expect("empty page needs no disk admission");
|
||||
assert!(empty.exhausted);
|
||||
}
|
||||
assert_eq!(Arc::strong_count(semaphore), owners, "cancelled entry points release the queue ownership");
|
||||
drop(held);
|
||||
let result = timeout(Duration::from_secs(5), store.clone().list_path(&opts))
|
||||
.await
|
||||
.expect("failed walk terminates");
|
||||
assert_eq!(
|
||||
result.err(),
|
||||
Some(DiskError::ErasureReadQuorum.into()),
|
||||
"offline drives retain their quorum error"
|
||||
);
|
||||
let next = timeout(Duration::from_secs(5), store.ctx.acquire_list_read_budget(usize::MAX))
|
||||
.await
|
||||
.expect("failed and cancelled walks release their grants")
|
||||
.expect("budget remains open");
|
||||
assert!(next.is_some());
|
||||
}
|
||||
|
||||
fn versioned_object_metadata(bucket: &str, name: &str, versions: &[(u128, i64)], slot: usize) -> Bytes {
|
||||
let mut meta = FileMeta::new();
|
||||
for (id, modified) in versions {
|
||||
@@ -6929,6 +7071,155 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_identifier_tree_filters_old_peer_rows_before_metadata_resolution() {
|
||||
let prefix = "table-catalog/namespaces/";
|
||||
let leaf = "table-entry.json";
|
||||
let corrupt = |name: &str| {
|
||||
MetaCacheEntries(vec![
|
||||
Some(MetaCacheEntry {
|
||||
name: format!("{prefix}{name}"),
|
||||
metadata: b"corrupt metadata".to_vec(),
|
||||
..Default::default()
|
||||
});
|
||||
4
|
||||
])
|
||||
};
|
||||
let report = "sales/tables/orders/maintenance/hash/jobs/job.json";
|
||||
let errors = vec![None; 4];
|
||||
assert!(
|
||||
SetDisks::resolve_authoritative_listed_entry(RUSTFS_META_BUCKET, None, corrupt(report), &errors, 4, 2,).is_err(),
|
||||
"the control must actually parse and reject the corrupt metadata"
|
||||
);
|
||||
assert!(
|
||||
SetDisks::resolve_authoritative_listed_entry(
|
||||
RUSTFS_META_BUCKET,
|
||||
Some((prefix, leaf)),
|
||||
corrupt(report),
|
||||
&errors,
|
||||
4,
|
||||
2,
|
||||
)
|
||||
.expect("an old peer's unrelated report is outside the selected scan")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
SetDisks::resolve_authoritative_listed_entry(
|
||||
RUSTFS_META_BUCKET,
|
||||
Some((prefix, leaf)),
|
||||
corrupt("sales/tables/orders/maintenance/tables/owner/table-entry.json"),
|
||||
&errors,
|
||||
4,
|
||||
2,
|
||||
)
|
||||
.is_err(),
|
||||
"a matching nested owner must still fail closed on corruption"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_identifier_tree_pages_preserve_nested_owners_and_read_quorum() {
|
||||
let (dirs, store) = single_set_store(4).await;
|
||||
let prefix = "table-catalog/identifier-tree/";
|
||||
let leaf = "table-entry.json";
|
||||
for (root, selected) in [
|
||||
("table-catalog/identifier-tree", leaf),
|
||||
(prefix, "../entry.json"),
|
||||
(prefix, "table-entry"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, root, selected, None, 2)
|
||||
.await,
|
||||
Err(StorageError::InvalidArgument(..))
|
||||
));
|
||||
}
|
||||
let names = [
|
||||
format!("{prefix}a/tables/first/{leaf}"),
|
||||
format!("{prefix}sales/tables/orders/maintenance/tables/owner/{leaf}"),
|
||||
format!("{prefix}z/tables/last/{leaf}"),
|
||||
];
|
||||
for name in &names {
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, name, |slot| Some(unversioned_catalog_metadata(name, slot))).await;
|
||||
}
|
||||
for name in [
|
||||
format!("{prefix}sales"),
|
||||
format!("{prefix}sales/tables/orders/maintenance/hash/jobs/job.json"),
|
||||
] {
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &name, |_| Some(b"corrupt noncandidate metadata".to_vec())).await;
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.list_unversioned_objects_authoritative(RUSTFS_META_BUCKET, prefix, None, 100)
|
||||
.await
|
||||
.is_err(),
|
||||
"the ordinary authoritative scan must expose the corrupt control objects"
|
||||
);
|
||||
let first = store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||
.await
|
||||
.expect("noncandidate metadata must not be read");
|
||||
assert!(first.is_truncated);
|
||||
assert_eq!(
|
||||
first.objects.iter().map(|object| &object.name).collect::<Vec<_>>(),
|
||||
names[..2].iter().collect::<Vec<_>>()
|
||||
);
|
||||
let second = store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, first.next_marker, 2)
|
||||
.await
|
||||
.expect("the next page must retain the final owner");
|
||||
assert!(!second.is_truncated);
|
||||
assert_eq!(second.objects.iter().map(|object| &object.name).collect::<Vec<_>>(), vec![&names[2]]);
|
||||
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |_| Some(b"corrupt selected metadata".to_vec())).await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||
.await
|
||||
.is_err(),
|
||||
"corruption at a selected key cannot be skipped to fill the page"
|
||||
);
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||
Some(unversioned_catalog_metadata(&names[0], slot))
|
||||
})
|
||||
.await;
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||
Some(versioned_object_metadata(RUSTFS_META_BUCKET, &names[0], &[(2, 1_700_000_002)], slot).to_vec())
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||
.await
|
||||
.is_err(),
|
||||
"a versioned ownership entry cannot be accepted as an unversioned proof"
|
||||
);
|
||||
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||
Some(unversioned_catalog_metadata(&names[0], slot))
|
||||
})
|
||||
.await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for disk in set.disks.write().await.iter_mut().skip(1) {
|
||||
*disk = None;
|
||||
}
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.clone()
|
||||
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||
.await,
|
||||
Err(StorageError::ErasureReadQuorum)
|
||||
),
|
||||
"unavailable replicas cannot prove owner absence"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::bucket::utils::check_put_object_part_args;
|
||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
|
||||
use crate::config::storageclass;
|
||||
use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
|
||||
use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaSaveGate};
|
||||
use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -480,9 +480,13 @@ pub struct ECStore {
|
||||
/// Serializes full-document pool metadata saves and retains a fail-closed
|
||||
/// write block after startup observes an unreadable replica.
|
||||
///
|
||||
/// Immutable admission proofs share authority; publication, recovery and
|
||||
/// authority changes retain exclusive access through their existing I/O.
|
||||
/// Each writer publishes its final state on release, and health
|
||||
/// inspection reads that committed state without acquiring the gate.
|
||||
/// Lock order: acquire `pool_meta_save_gate`, then the distributed
|
||||
/// `pool.bin` fence, then clone `pool_meta` under a short read lock.
|
||||
pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
|
||||
pub(crate) pool_meta_save_gate: PoolMetaSaveGate,
|
||||
/// Per-instance runtime state (Phase 5, backlog#939).
|
||||
///
|
||||
/// Carries this instance's identity/runtime out of the process globals so
|
||||
@@ -555,7 +559,6 @@ impl Default for ScannerDataMovementPauseStatus {
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct PoolMetaWriteGateStatus {
|
||||
pub writes_ready: bool,
|
||||
pub check_timed_out: bool,
|
||||
pub write_blocked: bool,
|
||||
pub transaction_aborted: bool,
|
||||
pub pool_meta_absent: bool,
|
||||
@@ -571,7 +574,6 @@ impl Default for PoolMetaWriteGateStatus {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
writes_ready: true,
|
||||
check_timed_out: false,
|
||||
write_blocked: false,
|
||||
transaction_aborted: false,
|
||||
pool_meta_absent: false,
|
||||
@@ -1118,6 +1120,20 @@ impl ECStore {
|
||||
Some(epoch)
|
||||
}
|
||||
|
||||
/// Namespace cleanup edits an existing usage baseline; it does not certify
|
||||
/// a scan. Retain the movement fence without requiring scanning to resume.
|
||||
pub(crate) async fn data_usage_namespace_cleanup_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
|
||||
let operation_guard = self.ctx.data_movement_operation_gate().read_owned().await;
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
return None;
|
||||
}
|
||||
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
|
||||
}
|
||||
|
||||
pub(crate) async fn data_usage_namespace_cleanup_epoch(&self) -> Option<u64> {
|
||||
self.data_usage_namespace_cleanup_guard().await.map(|(_, epoch)| epoch)
|
||||
}
|
||||
|
||||
/// Acquire a storage-owned read admission for a coordinator's final
|
||||
/// scanner publication. The guard remains in the context's lease table,
|
||||
/// so a local movement writer cannot pass the peer while its authoritative
|
||||
@@ -1936,7 +1952,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
};
|
||||
@@ -2011,7 +2027,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
|
||||
@@ -2133,7 +2133,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
|
||||
@@ -4937,6 +4937,87 @@ impl ECStore {
|
||||
Ok((idx, result))
|
||||
}
|
||||
|
||||
/// Compare the logical revision across all pools before publishing a
|
||||
/// namespace-only usage update to a writable pool. A suspended source is
|
||||
/// retained for decommission to reconcile against the newer target.
|
||||
pub(crate) async fn put_usage_snapshot_for_namespace_cleanup(
|
||||
&self,
|
||||
object: &str,
|
||||
expected_revision: Option<&str>,
|
||||
data: &mut PutObjReader,
|
||||
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<ObjectInfo> {
|
||||
// The caller owns the bucket namespace and movement guards. Acquire
|
||||
// the fixed object domain, then every physical domain in pool order.
|
||||
let guards = self
|
||||
.acquire_all_physical_object_write_locks("usage_namespace_cleanup", RUSTFS_META_BUCKET, object)
|
||||
.await?;
|
||||
let mut opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
for guard in &guards {
|
||||
guard.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(guard) = namespace_guard {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let source = match self
|
||||
.get_latest_object_info_for_conditional_write(RUSTFS_META_BUCKET, object, &opts)
|
||||
.await
|
||||
.and_then(|(info, idx)| resolve_latest_object_access(RUSTFS_META_BUCKET, object, info, idx, &opts))
|
||||
{
|
||||
Ok((info, _)) => Some(info),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if source.as_ref().and_then(|info| info.etag.as_deref()) != expected_revision {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if source.as_ref().is_some_and(|info| {
|
||||
info.version_id.is_some_and(|version| !version.is_nil()) || info.etag.as_ref().is_none_or(String::is_empty)
|
||||
}) {
|
||||
return Err(StorageError::other("usage cleanup requires an unversioned snapshot with a revision"));
|
||||
}
|
||||
let target = self.get_pool_idx_no_lock(RUSTFS_META_BUCKET, object, data.size()).await?;
|
||||
let target_info = match self.pools[target].get_object_info(RUSTFS_META_BUCKET, object, &opts).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
opts.http_preconditions = Some(match target_info.as_ref() {
|
||||
Some(info) => crate::storage_api_contracts::object::HTTPPreconditions {
|
||||
if_match: Some(
|
||||
info.etag
|
||||
.as_ref()
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.ok_or_else(|| StorageError::other("usage cleanup target has no revision"))?,
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
None => crate::storage_api_contracts::object::HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
});
|
||||
let latest = source
|
||||
.iter()
|
||||
.chain(target_info.iter())
|
||||
.filter_map(|info| info.mod_time)
|
||||
.fold(OffsetDateTime::now_utc(), std::cmp::max);
|
||||
opts.mod_time = Some(
|
||||
latest
|
||||
.checked_add(time::Duration::nanoseconds(1))
|
||||
.ok_or_else(|| StorageError::other("usage cleanup snapshot timestamp is exhausted"))?,
|
||||
);
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
self.pools[target].put_object(RUSTFS_META_BUCKET, object, data, &opts).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
pub(super) async fn handle_put_object(
|
||||
@@ -7005,7 +7086,7 @@ mod tests {
|
||||
};
|
||||
use crate::set_disk::same_distributed_lock_domain;
|
||||
use crate::set_disk::{SetDisks, disk_call_counters};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::bucket::{BucketOptions, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use crate::storage_api_contracts::list::ListOperations as _;
|
||||
use bytes::Bytes;
|
||||
@@ -10003,7 +10084,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
@@ -10095,7 +10176,7 @@ mod tests {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
@@ -10347,6 +10428,296 @@ mod tests {
|
||||
multipool_delete_test_store(bucket, true).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_fresh_bucket_after_canceled_decommission() {
|
||||
let (_dirs, store) = multipool_version_test_store("canceled-decommission-existing").await;
|
||||
let mut canceled = prepared_pool_test_status(1, false);
|
||||
canceled.decommission = Some(PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: vec![prepared_pool_test_status(0, false), canceled],
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.handle_make_bucket("canceled-decommission-fresh", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("canceled decommission must not block creation of a new bucket");
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
"scanner publication must remain fenced"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_bucket_after_failed_decommission() {
|
||||
let (_dirs, store) = multipool_version_test_store("decommission-existing-bucket").await;
|
||||
let bucket = "decommission-new-bucket";
|
||||
let snapshot_time = std::time::SystemTime::now() - Duration::from_secs(60);
|
||||
let mut snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(snapshot_time),
|
||||
scanner_cycle: Some(37),
|
||||
scanner_epoch: Some(2),
|
||||
buckets_count: 1,
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.to_string(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 42,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.to_string(), 42);
|
||||
snapshot.calculate_totals();
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("store a stale bucket-usage baseline before the failed decommission");
|
||||
let (old_snapshot, usage_owner) = store
|
||||
.get_latest_accessible_object_info_with_idx(
|
||||
RUSTFS_META_BUCKET,
|
||||
crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("identify the pool holding the old usage baseline");
|
||||
let mut pool_statuses = vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, false)];
|
||||
pool_statuses[usage_owner].decommission = Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: pool_statuses,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("a failed decommission must not block creation of an unrelated bucket");
|
||||
store
|
||||
.handle_get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("the created bucket must be visible");
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("load the cleaned usage baseline");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket), "the new bucket must not inherit old usage");
|
||||
assert_eq!(usage.scanner_cycle, Some(37), "namespace cleanup must not certify a new scan");
|
||||
assert_eq!(usage.scanner_epoch, Some(2), "namespace cleanup must preserve the scanner authority");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (_, current_owner) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the new snapshot authority");
|
||||
assert_ne!(current_owner, usage_owner, "cleanup must publish to a writable pool");
|
||||
let retained_source = store.pools[usage_owner]
|
||||
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("decommission retains its original source for reconciliation");
|
||||
assert_eq!(retained_source.etag, old_snapshot.etag);
|
||||
let stale_update = store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
old_snapshot.etag.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"stale snapshot".to_vec()),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(stale_update, Err(StorageError::PreconditionFailed)));
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("a stale source CAS must not replace the writable target");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket));
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
"scanner publication must remain fenced"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_create_retains_usage_cleanup_namespace_owner() {
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-cancellation-existing").await;
|
||||
let bucket = "cleanup-cancellation-new";
|
||||
let mut snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.to_owned(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 42,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.to_owned(), 42);
|
||||
snapshot.calculate_totals();
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the stale namespace usage");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (_, source) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("find the source snapshot");
|
||||
let mut pools = vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, false)];
|
||||
pools[source].decommission = Some(PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools,
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::AfterNamespace);
|
||||
let create_store = store.clone();
|
||||
let create = tokio::spawn(async move { create_store.handle_make_bucket(bucket, &MakeBucketOptions::default()).await });
|
||||
tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("cleanup reaches its physical commit with the namespace held");
|
||||
create.abort();
|
||||
assert!(create.await.expect_err("the request waiter was canceled").is_cancelled());
|
||||
let namespace = store
|
||||
.new_ns_lock(bucket, bucket)
|
||||
.await
|
||||
.expect("open the bucket namespace lock");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), namespace.get_write_lock(Duration::from_secs(1)))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancellation must not release the namespace before its cleanup finishes"
|
||||
);
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
tokio::time::timeout(Duration::from_secs(10), store.handle_make_bucket(bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("the resumed create must not remain blocked")
|
||||
.expect("a new request resumes after the old cleanup releases its namespace");
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("read the cleaned baseline");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket));
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_cleanup_rejects_an_unreadable_pool() {
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-unreadable-pool").await;
|
||||
let snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the usage baseline");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (before, source) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("find the baseline owner");
|
||||
let unavailable = 1 - source;
|
||||
for disk in store.pools[unavailable]
|
||||
.get_disks_by_key(object)
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
{
|
||||
disk.close().await.expect("stop disk health monitoring");
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
let update = store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
before.etag.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"unproven authority".to_vec()),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(update.is_err(), "an unreadable pool may hold a newer snapshot");
|
||||
let after = store.pools[source]
|
||||
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the readable baseline remains intact");
|
||||
assert_eq!(after.etag, before.etag);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_cleanup_rejects_namespace_loss_before_physical_commit() {
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
|
||||
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-fence-existing").await;
|
||||
let snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the protected baseline");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (before, _) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the baseline revision");
|
||||
let ttl = Duration::from_secs(60);
|
||||
let namespace = NamespaceLock::new("usage-cleanup-commit-fence".to_owned(), Arc::new(LocalClient::new()));
|
||||
let request = LockRequest::new(ObjectKey::new("cleanup-fence-existing", ""), LockType::Exclusive, "usage-cleanup")
|
||||
.with_acquire_timeout(Duration::from_secs(1))
|
||||
.with_ttl(ttl)
|
||||
.with_refresh_interval(ttl);
|
||||
let guard = Arc::new(
|
||||
namespace
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("acquire the namespace lease")
|
||||
.expect("namespace lease is available"),
|
||||
);
|
||||
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::AfterNamespace);
|
||||
let update_store = store.clone();
|
||||
let update_guard = guard.clone();
|
||||
let revision = before.etag.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
let _movement = update_store
|
||||
.data_usage_namespace_cleanup_guard()
|
||||
.await
|
||||
.expect("admit the namespace-only update");
|
||||
update_store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
revision.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"must not commit".to_vec()),
|
||||
Some(update_guard.as_ref()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("the update reaches its commit barrier");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(ttl + Duration::from_millis(1)).await;
|
||||
assert!(guard.is_lock_lost(), "the parent namespace lease has expired");
|
||||
tokio::time::resume();
|
||||
barrier.release();
|
||||
assert!(update.await.expect("the cleanup task completes").is_err());
|
||||
let (after, _) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the original baseline remains readable");
|
||||
assert_eq!(after.etag, before.etag, "a lost namespace lease must fence the physical write");
|
||||
}
|
||||
|
||||
async fn multipool_delete_test_store(bucket: &str, versioned: bool) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
|
||||
@@ -58,7 +58,7 @@ pub enum Error {
|
||||
TaskExecutionFailed { message: String },
|
||||
|
||||
/// Only the qualified MRF Deep check may produce this terminal result.
|
||||
#[error("Heal task execution failed: Missing exact storage proof for durable MRF repair {target}")]
|
||||
#[error("Independent verification unavailable for legacy object {target}; durable MRF responsibility retained")]
|
||||
LegacyMrfUnverified { target: String },
|
||||
|
||||
#[error("Replacement failure could not be persisted: {failure}; persistence error: {persistence}")]
|
||||
|
||||
@@ -292,6 +292,20 @@ impl HealManager {
|
||||
"Heal scheduler task completed"
|
||||
);
|
||||
}
|
||||
Err(e @ Error::LegacyMrfUnverified { .. }) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "verification_unavailable",
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
detail = %e,
|
||||
"Heal attempt completed without independent verification"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
let will_retry = retry_request.is_some();
|
||||
if will_retry {
|
||||
@@ -483,6 +497,9 @@ impl HealManager {
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
|
||||
stats.update_unverified_task_completion();
|
||||
}
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
|
||||
|
||||
@@ -1106,6 +1106,22 @@ impl HealStorageAPI for MockStorage {
|
||||
if bucket.starts_with("legacy-ambiguity-scheduler-") {
|
||||
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
|
||||
}
|
||||
if bucket == "legacy-held-pending-scheduler" {
|
||||
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
|
||||
state: "ok".to_owned(),
|
||||
..Default::default()
|
||||
}];
|
||||
return Ok((
|
||||
HealResultItem {
|
||||
legacy_identity_unverified: true,
|
||||
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
|
||||
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
|
||||
after: rustfs_madmin::heal_commands::Infos { drives },
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
));
|
||||
}
|
||||
if bucket == "completed-retention-failed" {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "retention fixture failure".to_string(),
|
||||
@@ -2153,6 +2169,53 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
|
||||
use crate::heal::failure_logging::test_support::FailureLogCapture;
|
||||
|
||||
let capture = FailureLogCapture::default();
|
||||
let _subscriber = tracing::subscriber::set_default(capture.clone());
|
||||
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
|
||||
let bucket = target.bucket.clone();
|
||||
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::object(bucket.to_string(), target.object.to_string(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(anchor.bucket_incarnation_id);
|
||||
request.options.pool_index = Some(1);
|
||||
request.options.set_index = Some(2);
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
|
||||
.await
|
||||
.expect("admit durable MRF check");
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while manager.get_statistics().await.unverified_tasks == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("publish completed unverified attempt");
|
||||
let stats = manager.get_statistics().await;
|
||||
assert_eq!(stats.unverified_tasks, 1);
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.total_objects_healed, 0);
|
||||
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
|
||||
assert_eq!(report.status, HealTaskStatus::Completed);
|
||||
let outcome = report.outcome.expect("retained unknown outcome");
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert!(rustfs_common::mrf_channel::take_mrf_verified_repair_events_for(&bucket).is_empty());
|
||||
let held = rustfs_common::mrf_channel::take_mrf_unverified_legacy_events_for(&bucket);
|
||||
assert_eq!(held.len(), 1);
|
||||
assert_eq!(held[0].anchor, anchor);
|
||||
assert!(manager.retrying_heals.lock().await.is_empty());
|
||||
assert_eq!(capture.count(tracing::Level::ERROR, Some("rustfs::heal::manager")), 0);
|
||||
assert_eq!(capture.count(tracing::Level::WARN, Some("rustfs::heal::manager")), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_mrf_notice_publishes_only_verified_positive_events() {
|
||||
use crate::heal::outcome::{HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome};
|
||||
|
||||
@@ -506,6 +506,9 @@ pub struct HealStatistics {
|
||||
pub successful_tasks: u64,
|
||||
/// Failed tasks
|
||||
pub failed_tasks: u64,
|
||||
/// Completed attempts without independent verification; not the current durable MRF backlog.
|
||||
#[serde(default)]
|
||||
pub unverified_tasks: u64,
|
||||
/// Running tasks
|
||||
pub running_tasks: u64,
|
||||
/// Total healed objects
|
||||
@@ -528,6 +531,7 @@ impl HealStatistics {
|
||||
total_tasks: 0,
|
||||
successful_tasks: 0,
|
||||
failed_tasks: 0,
|
||||
unverified_tasks: 0,
|
||||
running_tasks: 0,
|
||||
total_objects_healed: 0,
|
||||
total_bytes_healed: 0,
|
||||
@@ -549,6 +553,11 @@ impl HealStatistics {
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub(crate) fn update_unverified_task_completion(&mut self) {
|
||||
self.unverified_tasks = self.unverified_tasks.saturating_add(1);
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
|
||||
self.total_objects_healed += count;
|
||||
self.total_bytes_healed += bytes;
|
||||
@@ -915,6 +924,21 @@ mod tests {
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unverified_statistics_preserve_old_snapshot_compatibility() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.update_unverified_task_completion();
|
||||
assert_eq!(stats.unverified_tasks, 1);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
|
||||
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
|
||||
assert_eq!(current.unverified_tasks, 1);
|
||||
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
|
||||
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
|
||||
assert_eq!(old.unverified_tasks, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_update_task_completion_success() {
|
||||
let mut stats = HealStatistics::new();
|
||||
|
||||
@@ -291,7 +291,7 @@ pub enum HealTaskStatus {
|
||||
Running,
|
||||
/// Retrying after a recoverable failure
|
||||
Retrying { error: String, retry_attempt: u32 },
|
||||
/// Completed
|
||||
/// Execution completed; the outcome and durable MRF ledger retain any unverified responsibility.
|
||||
Completed,
|
||||
/// Failed
|
||||
Failed { error: String },
|
||||
@@ -1232,6 +1232,22 @@ impl HealTask {
|
||||
"Heal task completed"
|
||||
});
|
||||
}
|
||||
Err(Error::LegacyMrfUnverified { .. }) => {
|
||||
// The qualified Deep attempt finished, but its Unknown outcome
|
||||
// cannot discharge the separately owned durable responsibility.
|
||||
self.progress.write().await.mark_completed();
|
||||
*self.status.write().await = HealTaskStatus::Completed;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
state = "verification_unavailable",
|
||||
task_id = %self.id,
|
||||
heal_type = self.heal_type.kind_label(),
|
||||
"Heal attempt completed without independent verification"
|
||||
);
|
||||
}
|
||||
Err(Error::TaskCancelled) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Cancelled;
|
||||
@@ -1281,11 +1297,16 @@ impl HealTask {
|
||||
|
||||
let terminal_state = match &result {
|
||||
Ok(_) => "completed",
|
||||
Err(Error::LegacyMrfUnverified { .. }) => "verification_unavailable",
|
||||
Err(Error::TaskCancelled) => "cancelled",
|
||||
Err(Error::TaskTimeout) => "timed_out",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err());
|
||||
let terminal_error = result
|
||||
.as_ref()
|
||||
.err()
|
||||
.filter(|error| !matches!(error, Error::LegacyMrfUnverified { .. }));
|
||||
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), terminal_error);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
@@ -3285,6 +3285,29 @@ async fn durable_mrf_unverified_object_names_cannot_change_retry_classification(
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_legacy_verification_pending_is_a_completed_unverified_attempt() {
|
||||
let (task, _) = mrf_task_for_item("legacy-pending", healthy_legacy_heal_item());
|
||||
let error = task.execute().await.expect_err("unverified result must retain MRF ownership");
|
||||
assert!(matches!(error, Error::LegacyMrfUnverified { .. }));
|
||||
assert!(!error.is_recoverable_heal());
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
|
||||
let (status, detail) = outcome.legacy_status("finished", None);
|
||||
assert_eq!(status, "finished");
|
||||
assert!(
|
||||
detail
|
||||
.expect("visible unverified result")
|
||||
.contains("authoritative storage proof is unavailable")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_legacy_object_names_cannot_change_retry_classification() {
|
||||
for object in [
|
||||
@@ -3304,7 +3327,9 @@ async fn mrf_legacy_object_names_cannot_change_retry_classification() {
|
||||
assert!(matches!(error, Error::LegacyMrfUnverified { .. }));
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
format!("Heal task execution failed: Missing exact storage proof for durable MRF repair bucket-a/{object}")
|
||||
format!(
|
||||
"Independent verification unavailable for legacy object bucket-a/{object}; durable MRF responsibility retained"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3362,11 +3387,14 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
if case == "qualified" {
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
|
||||
let opts = storage.object_heal_opts.lock().expect("storage options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
|
||||
} else {
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Failed { .. }), "case={case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,78 @@ use storage_api::integration::{
|
||||
WriteCommitGuard, WriteCompletion,
|
||||
};
|
||||
|
||||
async fn complete_fixture_replicas(env: &TestECStoreEnv, bucket: &str, object: &str) {
|
||||
// TailDrained completes fan-out, but a successful quorum write may still
|
||||
// have minority failures. Fault injection needs a fully repaired baseline.
|
||||
let result = ECStoreHealStorage::new(env.ecstore.clone())
|
||||
.heal_object_with_receipt(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("complete protected fixture replicas before fault injection");
|
||||
assert!(result.error.is_none(), "fixture repair failed: {:?}", result.error);
|
||||
assert_eq!(result.item.after.drives.len(), env.disk_paths.len());
|
||||
assert!(
|
||||
result.item.after.drives.iter().all(|drive| drive.state == "ok"),
|
||||
"fixture requires every replica, not only read/write quorum: {:?}",
|
||||
result.item.after.drives
|
||||
);
|
||||
assert!(result.item.integrity_verified, "fixture must retain protected payload verification");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn mixed_batch_fixture_repairs_a_missing_minority_replica() {
|
||||
let root = tempfile::tempdir().expect("minority replica fixture");
|
||||
let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
|
||||
let bucket = "mixed-fixture-baseline";
|
||||
let object = "object";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let expected = vec![0x5a; 1024 * 1024 + 123];
|
||||
env.ecstore
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("quorum commit fixture");
|
||||
// Guarantee a minority failure regardless of whether the original write
|
||||
// succeeded on every disk, then repair it before any test fault is applied.
|
||||
let replica = env.disk_paths[0].join(bucket).join(object);
|
||||
if replica.exists() {
|
||||
tokio::fs::remove_dir_all(&replica)
|
||||
.await
|
||||
.expect("remove one complete replica");
|
||||
}
|
||||
complete_fixture_replicas(&env, bucket, object).await;
|
||||
let disks = env.ecstore.pools[0].get_disks(0);
|
||||
for disk in disks.disks.read().await.iter().flatten() {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("every fixture replica has metadata after completion");
|
||||
}
|
||||
let mut reader = env
|
||||
.ecstore
|
||||
.get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the completed fixture");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("completed fixture body");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_lock_reports_only_quorum_selection_and_deep_check_observations() {
|
||||
const CHILD_ENV: &str = "RUSTFS_TEST_DISABLED_LOCK_HEAL_CHILD";
|
||||
@@ -704,6 +776,7 @@ fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
|
||||
)
|
||||
.await
|
||||
.expect("commit complete fixture");
|
||||
complete_fixture_replicas(&env, bucket, object).await;
|
||||
for (slot, disk) in disks.iter().enumerate() {
|
||||
let meta = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
|
||||
@@ -66,11 +66,26 @@ pub fn select_effective_memory(host_total: u64, host_available: u64, cgroup: Opt
|
||||
|
||||
/// Resolves the effective memory from an already-refreshed system handle.
|
||||
///
|
||||
/// `cgroup_limits()` is computed fresh on each call and is only implemented on
|
||||
/// Linux (it returns `None` elsewhere), so non-Linux hosts always use host
|
||||
/// values.
|
||||
/// On Linux, the current process cgroup and its ancestors are read on each call,
|
||||
/// including limits imposed by systemd services. System-wide cgroup limits are
|
||||
/// used if process information is unavailable. Other platforms use host values.
|
||||
pub fn effective_memory_from_system(system: &System) -> EffectiveMemory {
|
||||
let cgroup = system.cgroup_limits().map(|limits| (limits.total_memory, limits.free_memory));
|
||||
#[cfg(target_os = "linux")]
|
||||
let limits = sysinfo::get_current_pid()
|
||||
.ok()
|
||||
.and_then(|pid| {
|
||||
let mut processes = System::new().ok()?;
|
||||
processes.refresh_processes_specifics(
|
||||
sysinfo::ProcessesToUpdate::Some(&[pid]),
|
||||
false,
|
||||
sysinfo::ProcessRefreshKind::nothing().with_memory(),
|
||||
);
|
||||
processes.process(pid)?.cgroup_limits()
|
||||
})
|
||||
.or_else(|| system.cgroup_limits());
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let limits = system.cgroup_limits();
|
||||
let cgroup = limits.map(|limits| (limits.total_memory, limits.free_memory));
|
||||
select_effective_memory(system.total_memory(), system.available_memory(), cgroup)
|
||||
}
|
||||
|
||||
@@ -89,6 +104,32 @@ mod tests {
|
||||
|
||||
const GIB: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn effective_memory_uses_current_process_cgroup() {
|
||||
use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System, get_current_pid};
|
||||
|
||||
let pid = get_current_pid().expect("current process ID");
|
||||
let mut system = System::new().expect("system memory handle");
|
||||
system.refresh_memory();
|
||||
let mut process_system = System::new().expect("process memory handle");
|
||||
process_system.refresh_processes_specifics(
|
||||
ProcessesToUpdate::Some(&[pid]),
|
||||
false,
|
||||
ProcessRefreshKind::nothing().with_memory(),
|
||||
);
|
||||
let limits = process_system.process(pid).expect("current process entry").cgroup_limits();
|
||||
let expected = select_effective_memory(
|
||||
system.total_memory(),
|
||||
system.available_memory(),
|
||||
limits.map(|limits| (limits.total_memory, limits.free_memory)),
|
||||
);
|
||||
let actual = super::effective_memory_from_system(&system);
|
||||
assert_eq!(actual.basis, expected.basis);
|
||||
assert_eq!(actual.total_bytes, expected.total_bytes);
|
||||
assert!(actual.available_bytes <= actual.total_bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_effective_memory_prefers_constraining_cgroup() {
|
||||
let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB)));
|
||||
|
||||
@@ -642,6 +642,7 @@ impl AdminAction {
|
||||
self,
|
||||
AdminAction::GetTableBucketAction
|
||||
| AdminAction::SetTableBucketAction
|
||||
| AdminAction::MigrateTableCatalogAction
|
||||
| AdminAction::GetTableNamespaceAction
|
||||
| AdminAction::SetTableNamespaceAction
|
||||
| AdminAction::UpdateTableNamespacePropertiesAction
|
||||
|
||||
@@ -1795,6 +1795,146 @@ mod test {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_catalog_migration_policy_matrix(data: &[u8], cases: &[(&str, AdminAction, bool)]) -> Result<()> {
|
||||
let policy = Policy::parse_config(data)?;
|
||||
let conditions = HashMap::new();
|
||||
let claims = HashMap::new();
|
||||
let groups = None;
|
||||
for (bucket, action, expected) in cases {
|
||||
let args = Args {
|
||||
account: "catalog-migration-operator",
|
||||
groups: &groups,
|
||||
action: Action::AdminAction(*action),
|
||||
bucket,
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "",
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
};
|
||||
assert_eq!(policy.is_allowed(&args).await, *expected, "{action:?} on warehouse {bucket:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_allow_resource_is_limited_to_the_requested_warehouse() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "admin:MigrateTableCatalog",
|
||||
"Resource": "arn:aws:s3:::warehouse-a"
|
||||
}]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
|
||||
("", AdminAction::MigrateTableCatalogAction, false),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_allow_not_resource_excludes_only_the_named_warehouse() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "admin:MigrateTableCatalog",
|
||||
"NotResource": "arn:aws:s3:::warehouse-b"
|
||||
}]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
|
||||
("warehouse-c", AdminAction::MigrateTableCatalogAction, true),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_explicit_deny_resource_does_not_spread_to_other_warehouses() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": "admin:*", "Resource": "*"},
|
||||
{"Effect": "Deny", "Action": "admin:MigrateTableCatalog", "Resource": "arn:aws:s3:::warehouse-b"}
|
||||
]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
|
||||
("warehouse-c", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_explicit_deny_not_resource_preserves_its_exception() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": "admin:*", "Resource": "*"},
|
||||
{"Effect": "Deny", "Action": "admin:MigrateTableCatalog", "NotResource": "arn:aws:s3:::warehouse-a"}
|
||||
]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
|
||||
("warehouse-c", AdminAction::MigrateTableCatalogAction, false),
|
||||
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_admin_wildcard_still_honors_the_requested_action_scope() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "admin:*",
|
||||
"Resource": "arn:aws:s3:::warehouse-a"
|
||||
}]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
|
||||
("", AdminAction::MigrateTableCatalogAction, false),
|
||||
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
|
||||
("", AdminAction::GetTableCatalogAction, true),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_migration_resource_free_admin_grants_keep_their_global_behavior() -> Result<()> {
|
||||
assert_catalog_migration_policy_matrix(
|
||||
br#"{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Action": "admin:MigrateTableCatalog"}]
|
||||
}"#,
|
||||
&[
|
||||
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
|
||||
("warehouse-b", AdminAction::MigrateTableCatalogAction, true),
|
||||
("", AdminAction::MigrateTableCatalogAction, true),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_table_admin_action_with_resource_is_limited_to_bucket() -> Result<()> {
|
||||
use crate::policy::action::{Action, AdminAction};
|
||||
|
||||
@@ -64,6 +64,8 @@ async-compression = { workspace = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
async-trait.workspace = true
|
||||
arc-swap.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
base64-simd.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
crc-fast.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
@@ -73,6 +75,9 @@ rustfs-storage-api.workspace = true
|
||||
futures = { workspace = true }
|
||||
futures-core = { workspace = true }
|
||||
flate2.workspace = true
|
||||
brotli.workspace = true
|
||||
lz4.workspace = true
|
||||
lz4_flex.workspace = true
|
||||
http.workspace = true
|
||||
memchr.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
|
||||
@@ -27,6 +27,7 @@ mod csv_input;
|
||||
mod input_stream;
|
||||
mod metrics;
|
||||
pub mod object_store;
|
||||
pub mod parquet_budget;
|
||||
pub mod query;
|
||||
pub mod server;
|
||||
mod storage_api;
|
||||
@@ -218,6 +219,14 @@ fn classify_select_error_source(err: &(dyn StdError + 'static)) -> Option<Select
|
||||
if let Some(error) = err.downcast_ref::<SelectError>() {
|
||||
return Some(error.clone());
|
||||
}
|
||||
if let Some(error) = err.downcast_ref::<parquet_budget::ParquetBudgetError>() {
|
||||
return Some(match error {
|
||||
parquet_budget::ParquetBudgetError::Exceeded => SelectError::ResourceExhausted,
|
||||
parquet_budget::ParquetBudgetError::Invalid(_) | parquet_budget::ParquetBudgetError::Codec(_) => {
|
||||
SelectError::ParquetParsingError
|
||||
}
|
||||
});
|
||||
}
|
||||
if let Some(error) = err.downcast_ref::<object_store::SelectObjectStoreError>() {
|
||||
return Some(error.select_error());
|
||||
}
|
||||
@@ -332,9 +341,32 @@ impl Display for ResolvedTable {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use datafusion::common::DataFusionError;
|
||||
use datafusion::sql::sqlparser::parser::ParserError;
|
||||
|
||||
#[test]
|
||||
fn parquet_admission_errors_preserve_select_error_codes() {
|
||||
for (source, expected) in [
|
||||
(parquet_budget::ParquetBudgetError::Exceeded, SelectError::ResourceExhausted),
|
||||
(
|
||||
parquet_budget::ParquetBudgetError::Invalid("malformed footer".into()),
|
||||
SelectError::ParquetParsingError,
|
||||
),
|
||||
(
|
||||
parquet_budget::ParquetBudgetError::Codec(std::io::Error::from(std::io::ErrorKind::InvalidData)),
|
||||
SelectError::ParquetParsingError,
|
||||
),
|
||||
] {
|
||||
let error = QueryError::from(DataFusionError::External(Box::new(source)));
|
||||
assert_eq!(error.select_error(), expected);
|
||||
}
|
||||
let nested = QueryError::from(DataFusionError::ParquetError(Box::new(ParquetError::External(Box::new(
|
||||
parquet_budget::ParquetBudgetError::Exceeded,
|
||||
)))));
|
||||
assert_eq!(nested.select_error(), SelectError::ResourceExhausted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_error_display() {
|
||||
let err = QueryError::NotImplemented {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -424,7 +424,8 @@ impl SimpleQueryDispatcher {
|
||||
tracker: &QueryExecutionTracker,
|
||||
) -> QueryResult<MetadataProvider> {
|
||||
if self.input.request.input_serialization.parquet.is_some() {
|
||||
let provider = ParquetSelectTable::try_new(session.inner(), self.input.as_ref()).await?;
|
||||
let query_guard = tracker.query_guard().ok_or(QueryError::Cancel)?;
|
||||
let provider = ParquetSelectTable::try_new(session.inner(), self.input.as_ref(), Some(query_guard)).await?;
|
||||
let current_session_table_provider = self.build_table_handle_provider()?;
|
||||
let metadata_provider =
|
||||
MetadataProvider::new(provider, current_session_table_provider, self.func_manager.clone(), session.clone());
|
||||
|
||||
@@ -27,7 +27,10 @@ use datafusion::{
|
||||
datasource::{
|
||||
TableProvider,
|
||||
listing::{ListingTableUrl, PartitionedFile},
|
||||
physical_plan::{FileScanConfigBuilder, ParquetSource, parquet::ParquetAccessPlan},
|
||||
physical_plan::{
|
||||
FileScanConfigBuilder, ParquetSource,
|
||||
parquet::{ParquetAccessPlan, ParquetFileReaderFactory},
|
||||
},
|
||||
source::DataSourceExec,
|
||||
},
|
||||
execution::{memory_pool::MemoryLimit, object_store::ObjectStoreUrl},
|
||||
@@ -40,6 +43,7 @@ use datafusion::{
|
||||
},
|
||||
physical_plan::{
|
||||
ExecutionPlan, SendableRecordBatchStream,
|
||||
metrics::ExecutionPlanMetricsSet,
|
||||
stream::RecordBatchStreamAdapter,
|
||||
streaming::{PartitionStream, StreamingTableExec},
|
||||
},
|
||||
@@ -48,6 +52,8 @@ use futures::{FutureExt, TryFutureExt, TryStreamExt, future::BoxFuture, stream};
|
||||
use rustfs_s3select_api::{
|
||||
QueryResult,
|
||||
object_store::{SelectScanRange, scan_range_from_bounds},
|
||||
parquet_budget::{ParquetBudgetError, page_decode_budget, validate_footer},
|
||||
query::session::QueryExecutionGuard,
|
||||
};
|
||||
use s3s::dto::SelectObjectContentInput;
|
||||
|
||||
@@ -59,6 +65,7 @@ pub struct ParquetSelectTable {
|
||||
object_size: u64,
|
||||
access_plan: Option<Arc<ParquetAccessPlan>>,
|
||||
expanded_row_budget: usize,
|
||||
reader_factory: Arc<BudgetParquetReaderFactory>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ParquetSelectTable {
|
||||
@@ -72,11 +79,30 @@ impl fmt::Debug for ParquetSelectTable {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ObjectStoreParquetReader {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
path: Path,
|
||||
file_size: u64,
|
||||
metadata_read_limit: usize,
|
||||
metadata_decode_limit: usize,
|
||||
metadata: Option<Arc<ParquetMetaData>>,
|
||||
query_guard: Option<QueryExecutionGuard>,
|
||||
}
|
||||
|
||||
async fn validate_in_worker<T: Send + 'static>(
|
||||
query_guard: Option<QueryExecutionGuard>,
|
||||
operation: impl FnOnce() -> Result<T, ParquetBudgetError> + Send + 'static,
|
||||
) -> ParquetResult<T> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Cancellation cannot release admission while the decoder preflight
|
||||
// still owns input buffers or codec allocations.
|
||||
let _query_guard = query_guard;
|
||||
operation()
|
||||
})
|
||||
.await
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))?
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))
|
||||
}
|
||||
|
||||
impl AsyncFileReader for ObjectStoreParquetReader {
|
||||
@@ -94,7 +120,16 @@ impl AsyncFileReader for ObjectStoreParquetReader {
|
||||
if total.is_none_or(|total| total > self.metadata_read_limit as u64) {
|
||||
return async { Err(ParquetError::External(Box::new(parquet_budget_error()))) }.boxed();
|
||||
}
|
||||
async move { self.store.get_ranges(&self.path, &ranges).await.map_err(parquet_store_error) }.boxed()
|
||||
async move {
|
||||
// Read requested ranges directly so sparse offsets cannot amplify
|
||||
// a small request into a large coalesced allocation.
|
||||
let mut bytes = Vec::with_capacity(ranges.len());
|
||||
for range in ranges {
|
||||
bytes.push(self.store.get_range(&self.path, range).await.map_err(parquet_store_error)?);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn get_metadata<'a>(
|
||||
@@ -102,6 +137,27 @@ impl AsyncFileReader for ObjectStoreParquetReader {
|
||||
options: Option<&'a ArrowReaderOptions>,
|
||||
) -> BoxFuture<'a, ParquetResult<Arc<ParquetMetaData>>> {
|
||||
async move {
|
||||
if let Some(metadata) = self.metadata.as_ref() {
|
||||
return Ok(Arc::clone(metadata));
|
||||
}
|
||||
let tail_start = self
|
||||
.file_size
|
||||
.checked_sub(8)
|
||||
.ok_or_else(|| ParquetError::EOF("Parquet footer is truncated".into()))?;
|
||||
let tail = self.get_bytes(tail_start..self.file_size).await?;
|
||||
let tail: &[u8; 8] = tail
|
||||
.as_ref()
|
||||
.try_into()
|
||||
.map_err(|_| ParquetError::EOF("Parquet footer is truncated".into()))?;
|
||||
let footer = datafusion::parquet::file::metadata::FooterTail::try_new(tail)?;
|
||||
let length =
|
||||
u64::try_from(footer.metadata_length()).map_err(|_| ParquetError::External(Box::new(parquet_budget_error())))?;
|
||||
let start = tail_start
|
||||
.checked_sub(length)
|
||||
.ok_or_else(|| ParquetError::EOF("Parquet metadata is truncated".into()))?;
|
||||
let bytes = self.get_bytes(start..tail_start).await?;
|
||||
let limit = self.metadata_decode_limit;
|
||||
validate_in_worker(self.query_guard.clone(), move || validate_footer(&bytes, limit)).await?;
|
||||
let metadata_options = options.map(|options| options.metadata_options().clone());
|
||||
let mut metadata_reader = ParquetMetaDataReader::new().with_metadata_options(metadata_options);
|
||||
|
||||
@@ -119,8 +175,84 @@ impl AsyncFileReader for ObjectStoreParquetReader {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BudgetParquetReaderFactory {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
path: Path,
|
||||
file_size: u64,
|
||||
metadata: Arc<ParquetMetaData>,
|
||||
read_limit: usize,
|
||||
}
|
||||
|
||||
impl ParquetFileReaderFactory for BudgetParquetReaderFactory {
|
||||
fn create_reader(
|
||||
&self,
|
||||
_partition_index: usize,
|
||||
file: PartitionedFile,
|
||||
_metadata_size_hint: Option<usize>,
|
||||
_metrics: &ExecutionPlanMetricsSet,
|
||||
) -> DFResult<Box<dyn AsyncFileReader + Send>> {
|
||||
if file.object_meta.location != self.path || file.object_meta.size != self.file_size {
|
||||
return Err(DataFusionError::Execution("Parquet reader input differs from its admitted object".into()));
|
||||
}
|
||||
Ok(Box::new(ObjectStoreParquetReader {
|
||||
store: Arc::clone(&self.store),
|
||||
path: self.path.clone(),
|
||||
file_size: self.file_size,
|
||||
metadata_read_limit: self.read_limit,
|
||||
metadata_decode_limit: self.read_limit,
|
||||
metadata: Some(Arc::clone(&self.metadata)),
|
||||
query_guard: None,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_parquet_pages(
|
||||
store: &Arc<dyn ObjectStore>,
|
||||
path: &Path,
|
||||
file_size: u64,
|
||||
metadata: &ParquetMetaData,
|
||||
plan: Option<&ParquetAccessPlan>,
|
||||
limit: usize,
|
||||
query_guard: &Option<QueryExecutionGuard>,
|
||||
) -> DFResult<()> {
|
||||
for (index, group) in metadata.row_groups().iter().enumerate() {
|
||||
if plan.is_some_and(|plan| !plan.should_scan(index)) {
|
||||
continue;
|
||||
}
|
||||
let mut resident = 0usize;
|
||||
for column in group.columns() {
|
||||
let start = u64::try_from(column.dictionary_page_offset().unwrap_or(column.data_page_offset()))
|
||||
.map_err(|_| parquet_budget_error())?;
|
||||
let length = u64::try_from(column.compressed_size()).map_err(|_| parquet_budget_error())?;
|
||||
let end = start
|
||||
.checked_add(length)
|
||||
.filter(|end| *end <= file_size)
|
||||
.ok_or_else(parquet_budget_error)?;
|
||||
if length > u64::try_from(limit).map_err(|_| parquet_budget_error())? {
|
||||
return Err(parquet_budget_error());
|
||||
}
|
||||
let bytes = store.get_range(path, start..end).await?;
|
||||
let declared = usize::try_from(column.uncompressed_size()).map_err(|_| parquet_budget_error())?;
|
||||
let codec = column.compression();
|
||||
let required = validate_in_worker(query_guard.clone(), move || page_decode_budget(&bytes, declared, limit, codec))
|
||||
.await
|
||||
.map_err(DataFusionError::from)?;
|
||||
resident = resident
|
||||
.checked_add(required)
|
||||
.filter(|total| *total <= limit)
|
||||
.ok_or_else(parquet_budget_error)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl ParquetSelectTable {
|
||||
pub async fn try_new(state: &dyn Session, input: &SelectObjectContentInput) -> QueryResult<Arc<dyn TableProvider>> {
|
||||
pub async fn try_new(
|
||||
state: &dyn Session,
|
||||
input: &SelectObjectContentInput,
|
||||
query_guard: Option<QueryExecutionGuard>,
|
||||
) -> QueryResult<Arc<dyn TableProvider>> {
|
||||
let table_path = ListingTableUrl::parse(format!("s3://{}/{}", input.bucket, input.key))?;
|
||||
let object_store_url = table_path.object_store();
|
||||
let object_location = Path::from(input.key.clone());
|
||||
@@ -136,9 +268,12 @@ impl ParquetSelectTable {
|
||||
};
|
||||
let reader = ObjectStoreParquetReader {
|
||||
store: Arc::clone(&store),
|
||||
path: object_location,
|
||||
path: object_location.clone(),
|
||||
file_size: object_meta.size,
|
||||
metadata_read_limit: (memory_budget / 16).max(1),
|
||||
metadata_decode_limit: (memory_budget / 4).max(1),
|
||||
metadata: None,
|
||||
query_guard: query_guard.clone(),
|
||||
};
|
||||
let builder = ParquetRecordBatchStreamBuilder::new(reader)
|
||||
.await
|
||||
@@ -153,11 +288,26 @@ impl ParquetSelectTable {
|
||||
builder.schema().metadata().clone(),
|
||||
));
|
||||
let metadata = Arc::clone(builder.metadata());
|
||||
// This is admission for valid Parquet declarations, not a replacement
|
||||
// for limits inside the footer parser or native compression codecs.
|
||||
let expanded_row_budget = (memory_budget / 16).max(1);
|
||||
let access_plan = parquet_access_plan(input, object_meta.size, metadata.as_ref())?;
|
||||
admit_parquet_decode(metadata.as_ref(), access_plan.as_deref(), memory_budget / 4)?;
|
||||
validate_parquet_pages(
|
||||
&store,
|
||||
&object_location,
|
||||
object_meta.size,
|
||||
&metadata,
|
||||
access_plan.as_deref(),
|
||||
memory_budget / 4,
|
||||
&query_guard,
|
||||
)
|
||||
.await?;
|
||||
let reader_factory = Arc::new(BudgetParquetReaderFactory {
|
||||
store,
|
||||
path: object_location,
|
||||
file_size: object_meta.size,
|
||||
metadata,
|
||||
read_limit: (memory_budget / 4).max(1),
|
||||
});
|
||||
|
||||
Ok(Arc::new(Self {
|
||||
schema,
|
||||
@@ -166,6 +316,7 @@ impl ParquetSelectTable {
|
||||
object_size: object_meta.size,
|
||||
access_plan,
|
||||
expanded_row_budget,
|
||||
reader_factory,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -202,7 +353,12 @@ impl TableProvider for ParquetSelectTable {
|
||||
));
|
||||
}
|
||||
let scan_limit = if filters.is_empty() { limit } else { None };
|
||||
let file_source = Arc::new(ParquetSource::new(Arc::clone(&self.schema)));
|
||||
let file_source = Arc::new(
|
||||
ParquetSource::new(Arc::clone(&self.schema))
|
||||
.with_parquet_file_reader_factory(self.reader_factory.clone())
|
||||
.with_enable_page_index(false)
|
||||
.with_bloom_filter_on_read(false),
|
||||
);
|
||||
let config = FileScanConfigBuilder::new(self.object_store_url.clone(), file_source)
|
||||
.with_file(self.partitioned_file())
|
||||
.with_projection_indices(projection.cloned())?
|
||||
@@ -439,6 +595,39 @@ pub(super) fn dictionary_parquet_for_test(value_bytes: usize, rows: usize) -> Ve
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn parquet_cancelled_preflight_retains_admission_until_worker_finishes() {
|
||||
let admission = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let permit = Arc::clone(&admission).acquire_owned().await.expect("initial admission");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let task = tokio::spawn(validate_in_worker(Some(Arc::new(permit)), move || {
|
||||
started_tx.send(()).expect("report running worker");
|
||||
release_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(5))
|
||||
.expect("worker release");
|
||||
Ok(())
|
||||
}));
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), started_rx)
|
||||
.await
|
||||
.expect("worker starts before deadline")
|
||||
.expect("worker started");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancelled request task").is_cancelled());
|
||||
assert!(Arc::clone(&admission).try_acquire_owned().is_err(), "running worker still owns admission");
|
||||
release_tx.send(()).expect("release worker");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
if Arc::clone(&admission).try_acquire_owned().is_ok() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker releases admission after completion");
|
||||
}
|
||||
use datafusion::{
|
||||
arrow::{
|
||||
array::Int32Array,
|
||||
@@ -519,7 +708,7 @@ mod tests {
|
||||
let context = parquet_session(store);
|
||||
let state = context.state();
|
||||
|
||||
let error = match ParquetSelectTable::try_new(&state, &parquet_input("missing.parquet")).await {
|
||||
let error = match ParquetSelectTable::try_new(&state, &parquet_input("missing.parquet"), None).await {
|
||||
Ok(_) => panic!("missing parquet object must fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
@@ -538,7 +727,7 @@ mod tests {
|
||||
let context = parquet_session(store);
|
||||
let state = context.state();
|
||||
|
||||
let error = match ParquetSelectTable::try_new(&state, &parquet_input(object.as_ref())).await {
|
||||
let error = match ParquetSelectTable::try_new(&state, &parquet_input(object.as_ref()), None).await {
|
||||
Ok(_) => panic!("corrupt parquet metadata must fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
@@ -581,7 +770,7 @@ mod tests {
|
||||
.put(&Path::from("dictionary.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &parquet_input("dictionary.parquet"))
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &parquet_input("dictionary.parquet"), None)
|
||||
.await
|
||||
.expect("valid dictionary metadata");
|
||||
context.register_table("source", table).expect("table");
|
||||
@@ -752,7 +941,7 @@ mod tests {
|
||||
start: Some(starts[0]),
|
||||
end: Some(starts[0]),
|
||||
});
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &input)
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &input, None)
|
||||
.await
|
||||
.expect("an unselected large row group must not reject the small range");
|
||||
context.register_table("source", table).expect("table");
|
||||
@@ -770,7 +959,7 @@ mod tests {
|
||||
start: Some(starts[1]),
|
||||
end: Some(starts[1]),
|
||||
});
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &input).await {
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &input, None).await {
|
||||
Ok(_) => panic!("the selected oversized row group must still be rejected before decoding"),
|
||||
Err(error) => error,
|
||||
};
|
||||
@@ -806,13 +995,209 @@ mod tests {
|
||||
.put(&Path::from("large.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &parquet_input("large.parquet")).await {
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &parquet_input("large.parquet"), None).await {
|
||||
Ok(_) => panic!("valid declared decoded size must be admitted before page decode"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("query memory budget"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_footer_collection_budget_is_checked_before_metadata_decode() {
|
||||
// A root schema with no leaves and 50 empty, valid row groups. The
|
||||
// encoded footer fits the read budget; its reservations do not.
|
||||
let mut footer = vec![0x15, 2, 0x19, 0x1c, 0x48, 4];
|
||||
footer.extend(b"root");
|
||||
footer.extend([0x15, 0, 0, 0x16, 0, 0x19, 0xfc, 50]);
|
||||
for _ in 0..50 {
|
||||
footer.extend([0x19, 0x0c, 0x16, 0, 0x16, 0, 0]);
|
||||
}
|
||||
footer.push(0);
|
||||
let decoded = ParquetMetaDataReader::decode_metadata(&footer).expect("valid compact footer fixture");
|
||||
assert_eq!(decoded.num_row_groups(), 50);
|
||||
let mut bytes = b"PAR1".to_vec();
|
||||
bytes.extend(&footer);
|
||||
bytes.extend(u32::try_from(footer.len()).expect("small footer").to_le_bytes());
|
||||
bytes.extend(b"PAR1");
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session_with_memory(store.clone(), 8192);
|
||||
store
|
||||
.put(&Path::from("footer.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &parquet_input("footer.parquet"), None).await {
|
||||
Ok(_) => panic!("footer collection reservations exceed the query budget"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_forged_footer_cannot_hide_oversized_compressed_pages() {
|
||||
use datafusion::parquet::file::metadata::ParquetMetaDataWriter;
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Utf8, false)]));
|
||||
let batch = RecordBatch::try_new(
|
||||
Arc::clone(&schema),
|
||||
vec![Arc::new(datafusion::arrow::array::StringArray::from(vec![
|
||||
"x".repeat(32 * 1024 * 1024),
|
||||
]))],
|
||||
)
|
||||
.expect("large compressible value");
|
||||
let properties = datafusion::parquet::file::properties::WriterProperties::builder()
|
||||
.set_dictionary_page_size_limit(64 * 1024 * 1024)
|
||||
.set_compression(datafusion::parquet::basic::Compression::ZSTD(Default::default()))
|
||||
.build();
|
||||
let mut encoded = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new(&mut encoded, schema, Some(properties)).expect("writer");
|
||||
writer.write(&batch).expect("compressed value");
|
||||
writer.close().expect("footer");
|
||||
let original = Bytes::from(encoded);
|
||||
let builder = ParquetRecordBatchReaderBuilder::try_new(original.clone()).expect("valid compressed fixture");
|
||||
let metadata = builder.metadata();
|
||||
let groups = metadata
|
||||
.row_groups()
|
||||
.iter()
|
||||
.map(|group| {
|
||||
let columns = group
|
||||
.columns()
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|column| {
|
||||
column
|
||||
.into_builder()
|
||||
.set_total_uncompressed_size(64)
|
||||
.build()
|
||||
.expect("forged column")
|
||||
})
|
||||
.collect();
|
||||
group
|
||||
.clone()
|
||||
.into_builder()
|
||||
.set_column_metadata(columns)
|
||||
.build()
|
||||
.expect("forged group")
|
||||
})
|
||||
.collect();
|
||||
let forged = metadata.as_ref().clone().into_builder().set_row_groups(groups).build();
|
||||
let footer_size = u32::from_le_bytes(
|
||||
original[original.len() - 8..original.len() - 4]
|
||||
.try_into()
|
||||
.expect("footer size"),
|
||||
);
|
||||
let mut bytes = original[..original.len() - 8 - usize::try_from(footer_size).expect("footer length")].to_vec();
|
||||
ParquetMetaDataWriter::new(&mut bytes, &forged)
|
||||
.finish()
|
||||
.expect("forged footer");
|
||||
assert!(bytes.len() < 256 * 1024, "compressed input remains small");
|
||||
admit_parquet_decode(&forged, None, 4 * 1024 * 1024).expect("footer-only admission misses the page size");
|
||||
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session_with_memory(store.clone(), 16 * 1024 * 1024);
|
||||
store
|
||||
.put(&Path::from("forged.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let result = ParquetSelectTable::try_new(&context.state(), &parquet_input("forged.parquet"), None).await;
|
||||
let error = match result {
|
||||
Ok(_) => panic!("actual query admission must reject oversized page buffers before decoding"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_legacy_lz4_checks_native_scratch_before_query_decode() {
|
||||
use datafusion::parquet::{basic::Compression, file::metadata::ParquetMetaDataWriter};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)]));
|
||||
let batch =
|
||||
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(datafusion::arrow::array::Int32Array::from(vec![42]))])
|
||||
.expect("one integer row");
|
||||
let properties = datafusion::parquet::file::properties::WriterProperties::builder()
|
||||
.set_dictionary_enabled(false)
|
||||
.build();
|
||||
let mut original = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new(&mut original, schema, Some(properties)).expect("writer");
|
||||
writer.write(&batch).expect("row");
|
||||
writer.close().expect("footer");
|
||||
let metadata = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(original))
|
||||
.expect("metadata")
|
||||
.metadata()
|
||||
.clone();
|
||||
// A valid legacy frame encoding the four little-endian bytes for 42.
|
||||
let payload = [2, 33, 76, 24, 5, 0, 0, 0, 0x40, 42, 0, 0, 0];
|
||||
let mut page = vec![0x15, 0, 0x15, 8, 0x15, 26, 0x2c, 0x15, 2, 0x15, 0, 0x15, 6, 0x15, 6, 0, 0];
|
||||
page.extend(payload);
|
||||
let column = metadata
|
||||
.row_group(0)
|
||||
.column(0)
|
||||
.clone()
|
||||
.into_builder()
|
||||
.set_compression(Compression::LZ4)
|
||||
.set_data_page_offset(4)
|
||||
.set_total_compressed_size(i64::try_from(page.len()).expect("small page"))
|
||||
.build()
|
||||
.expect("legacy column");
|
||||
let group = metadata
|
||||
.row_group(0)
|
||||
.clone()
|
||||
.into_builder()
|
||||
.set_column_metadata(vec![column])
|
||||
.build()
|
||||
.expect("row group");
|
||||
let metadata = metadata.as_ref().clone().into_builder().set_row_groups(vec![group]).build();
|
||||
let mut encoded = b"PAR1".to_vec();
|
||||
encoded.extend(page);
|
||||
ParquetMetaDataWriter::new(&mut encoded, &metadata)
|
||||
.finish()
|
||||
.expect("legacy footer");
|
||||
assert!(encoded.len() < 4096);
|
||||
let mut native = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(encoded.clone()))
|
||||
.expect("legacy metadata")
|
||||
.build()
|
||||
.expect("native reader");
|
||||
let decoded = native.next().expect("one batch").expect("legacy fallback decode");
|
||||
assert_eq!(
|
||||
decoded
|
||||
.column(0)
|
||||
.as_primitive::<datafusion::arrow::datatypes::Int32Type>()
|
||||
.value(0),
|
||||
42
|
||||
);
|
||||
let store = Arc::new(InMemory::new());
|
||||
store
|
||||
.put(&Path::from("legacy.parquet"), Bytes::from(encoded).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let tight = parquet_session_with_memory(store.clone(), 2 * 1024 * 1024);
|
||||
let result = ParquetSelectTable::try_new(&tight.state(), &parquet_input("legacy.parquet"), None).await;
|
||||
let error = match result {
|
||||
Ok(_) => panic!("legacy decoder scratch exceeds the query budget"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
||||
let roomy = parquet_session_with_memory(store, 128 * 1024 * 1024);
|
||||
let table = ParquetSelectTable::try_new(&roomy.state(), &parquet_input("legacy.parquet"), None)
|
||||
.await
|
||||
.expect("admitted legacy file");
|
||||
roomy.register_table("source", table).expect("table");
|
||||
let rows = roomy
|
||||
.sql("SELECT v FROM source")
|
||||
.await
|
||||
.expect("plan")
|
||||
.collect()
|
||||
.await
|
||||
.expect("query");
|
||||
assert_eq!(rows.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
|
||||
assert_eq!(
|
||||
rows[0]
|
||||
.column(0)
|
||||
.as_primitive::<datafusion::arrow::datatypes::Int32Type>()
|
||||
.value(0),
|
||||
42
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_nested_views_decode_through_the_actual_file_schema() {
|
||||
use datafusion::arrow::array::{ListArray, StringArray};
|
||||
|
||||
@@ -690,6 +690,19 @@ pub(super) enum ScannerPauseBacklogAttemptDecision {
|
||||
pub struct ScannerPauseBacklogReplicaId {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
/// Stable format identity; indices are only the current routing location.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub set_id: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogReplicaId {
|
||||
fn from_set(set: &SetDisks) -> Result<Self, EcstoreError> {
|
||||
Ok(Self {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
set_id: Some(set.persistent_set_id()?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
|
||||
@@ -706,9 +719,23 @@ impl ScannerPauseBacklogCommitRecord {
|
||||
|
||||
fn validate(&self) -> Result<(), String> {
|
||||
self.ledger.validate()?;
|
||||
if self.replicas.is_empty() || !self.replicas.windows(2).all(|pair| pair[0] < pair[1]) {
|
||||
if self.replicas.is_empty()
|
||||
|| !self
|
||||
.replicas
|
||||
.windows(2)
|
||||
.all(|pair| (pair[0].pool_index, pair[0].set_index) < (pair[1].pool_index, pair[1].set_index))
|
||||
{
|
||||
return Err("scanner pause backlog commit has invalid replica membership".to_string());
|
||||
}
|
||||
let mut identities = BTreeSet::new();
|
||||
if self
|
||||
.replicas
|
||||
.iter()
|
||||
.filter_map(|id| id.set_id)
|
||||
.any(|id| id.is_nil() || !identities.insert(id))
|
||||
{
|
||||
return Err("scanner pause backlog commit has invalid persistent set identities".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -747,6 +774,10 @@ impl ScannerPauseBacklogReplicaRecord {
|
||||
}
|
||||
if let Some(committed) = &self.committed {
|
||||
committed.validate()?;
|
||||
if committed.replicas.iter().any(|id| id.set_id.is_some()) && committed.replicas.iter().any(|id| id.set_id.is_none())
|
||||
{
|
||||
return Err("scanner pause backlog membership mixes positional and persistent identities".to_string());
|
||||
}
|
||||
if let Some(stable) = &self.stable {
|
||||
let stable_key = (stable.writer_epoch, stable.generation);
|
||||
let committed_key = (committed.ledger.writer_epoch, committed.ledger.generation);
|
||||
@@ -907,9 +938,19 @@ fn unix_now() -> u64 {
|
||||
}
|
||||
|
||||
async fn read_scanner_pause_backlog_replica(store: Arc<SetDisks>) -> ScannerPauseBacklogReplica {
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: store.pool_index,
|
||||
set_index: store.set_index,
|
||||
let id = match ScannerPauseBacklogReplicaId::from_set(&store) {
|
||||
Ok(id) => id,
|
||||
Err(error) => {
|
||||
return ScannerPauseBacklogReplica {
|
||||
id: ScannerPauseBacklogReplicaId {
|
||||
pool_index: store.pool_index,
|
||||
set_index: store.set_index,
|
||||
set_id: None,
|
||||
},
|
||||
revision: None,
|
||||
state: ScannerPauseBacklogReplicaState::Unavailable(error.to_string()),
|
||||
};
|
||||
}
|
||||
};
|
||||
let reader = match store
|
||||
.get_object_reader(
|
||||
@@ -1333,8 +1374,48 @@ fn select_scanner_pause_backlog_roll_forward(replicas: &[ScannerPauseBacklogRepl
|
||||
.then(|| newest.clone())
|
||||
}
|
||||
|
||||
/// Resolve stored membership against immutable set identities before checking
|
||||
/// commit proofs. A missing identity is never rebound to a new set at its old
|
||||
/// index. Legacy records retain their positional interpretation until an
|
||||
/// existing commit or rollback proof permits publishing the bound membership.
|
||||
fn normalize_scanner_pause_backlog_membership(
|
||||
replicas: &mut [ScannerPauseBacklogReplica],
|
||||
) -> Result<(), ScannerPauseBacklogSelectionError> {
|
||||
let mut by_location = HashMap::with_capacity(replicas.len());
|
||||
let mut by_identity = HashMap::with_capacity(replicas.len());
|
||||
for replica in replicas.iter() {
|
||||
let id = replica.id;
|
||||
if by_location.insert((id.pool_index, id.set_index), id).is_some()
|
||||
|| id
|
||||
.set_id
|
||||
.is_some_and(|identity| identity.is_nil() || by_identity.insert(identity, id).is_some())
|
||||
{
|
||||
return Err(ScannerPauseBacklogSelectionError::AuthorityConflict(
|
||||
"scanner pause backlog has duplicate or invalid storage identities".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
for replica in replicas {
|
||||
if let ScannerPauseBacklogReplicaState::Valid(record) = &mut replica.state
|
||||
&& let Some(commit) = record.committed.as_mut()
|
||||
{
|
||||
for member in &mut commit.replicas {
|
||||
let current = match member.set_id {
|
||||
Some(identity) => by_identity.get(&identity),
|
||||
None => by_location.get(&(member.pool_index, member.set_index)),
|
||||
};
|
||||
if let Some(current) = current {
|
||||
*member = *current;
|
||||
}
|
||||
}
|
||||
commit.replicas.sort_unstable();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn select_scanner_pause_backlog_replicas(
|
||||
replicas: Vec<ScannerPauseBacklogReplica>,
|
||||
mut replicas: Vec<ScannerPauseBacklogReplica>,
|
||||
) -> Result<LoadedScannerPauseBacklog, ScannerPauseBacklogSelectionFailure> {
|
||||
if replicas.is_empty() {
|
||||
return Err(scanner_pause_backlog_selection_failure(
|
||||
@@ -1354,6 +1435,9 @@ fn select_scanner_pause_backlog_replicas(
|
||||
)),
|
||||
));
|
||||
}
|
||||
if let Err(error) = normalize_scanner_pause_backlog_membership(&mut replicas) {
|
||||
return Err(scanner_pause_backlog_selection_failure(replicas, error));
|
||||
}
|
||||
|
||||
let replica_ids = scanner_pause_backlog_replica_ids(&replicas);
|
||||
let authoritative_commit = match select_scanner_pause_backlog_commit(&replicas, &replica_ids) {
|
||||
@@ -1512,12 +1596,13 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
) -> Result<(Vec<ScannerPauseBacklogReplica>, Vec<ScannerPauseBacklogReplica>), ScannerPauseBacklogRetirementError> {
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut decoded = Vec::with_capacity(replicas.len());
|
||||
let mut surviving = Vec::new();
|
||||
let mut writable_ids = BTreeSet::new();
|
||||
let mut has_source = false;
|
||||
for replica in replicas {
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: replica.pool_index,
|
||||
set_index: replica.set_index,
|
||||
set_id: replica.set_id,
|
||||
};
|
||||
if !ids.insert(id) {
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
@@ -1548,7 +1633,7 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
state,
|
||||
};
|
||||
if replica.pool_index != source_pool_index && replica.writable {
|
||||
surviving.push(decoded_replica.clone());
|
||||
writable_ids.insert(id);
|
||||
}
|
||||
decoded.push(decoded_replica);
|
||||
}
|
||||
@@ -1557,6 +1642,12 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
"scanner pause backlog retirement has no native source record",
|
||||
));
|
||||
}
|
||||
normalize_scanner_pause_backlog_membership(&mut decoded).map_err(scanner_pause_backlog_retirement_selection_error)?;
|
||||
let surviving = decoded
|
||||
.iter()
|
||||
.filter(|replica| writable_ids.contains(&replica.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
Ok((decoded, surviving))
|
||||
}
|
||||
|
||||
@@ -1874,11 +1965,9 @@ where
|
||||
let loaded_ids = loaded.replicas.iter().map(|replica| replica.id).collect::<BTreeSet<_>>();
|
||||
let writable_ids = writable
|
||||
.iter()
|
||||
.map(|set| ScannerPauseBacklogReplicaId {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
.map(|set| ScannerPauseBacklogReplicaId::from_set(set))
|
||||
.collect::<Result<BTreeSet<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
if writable_ids != loaded_ids {
|
||||
return Err("scanner pause backlog replica topology changed during commit".to_string());
|
||||
}
|
||||
@@ -1901,10 +1990,7 @@ where
|
||||
let results = tokio::spawn(async move {
|
||||
let mut results = Vec::with_capacity(writable.len());
|
||||
for set in writable {
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
};
|
||||
let id = ScannerPauseBacklogReplicaId::from_set(&set).map_err(|error| error.to_string())?;
|
||||
if only.as_ref().is_some_and(|only| !only.contains(&id)) {
|
||||
continue;
|
||||
}
|
||||
@@ -1928,10 +2014,10 @@ where
|
||||
break;
|
||||
}
|
||||
}
|
||||
results
|
||||
Ok::<_, String>(results)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| format!("scanner pause backlog publication owner failed: {err}"))?;
|
||||
.map_err(|err| format!("scanner pause backlog publication owner failed: {err}"))??;
|
||||
|
||||
let failures = results
|
||||
.iter()
|
||||
@@ -2444,6 +2530,142 @@ mod tests {
|
||||
(root, store)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority() {
|
||||
assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_pool_reindex_with_failed_membership_publication_preserves_authority() {
|
||||
assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(true);
|
||||
}
|
||||
|
||||
fn assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(fail_publication: bool) {
|
||||
run_native_retirement_test(async move || {
|
||||
use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault;
|
||||
|
||||
let (root, store) = native_pause_backlog_store(3, 1).await;
|
||||
let now = unix_now();
|
||||
let original = seed_native_expansion_ledger(&store, now).await;
|
||||
// Model bytes written by the previous positional-membership writer.
|
||||
for id in native_replica_ids(&store) {
|
||||
let (bytes, _) = native_replica_bytes(&store.pools[id.pool_index].disk_set[0]).await;
|
||||
let ScannerPauseBacklogReplicaState::Valid(mut record) = decode_scanner_pause_backlog_ledger(&bytes) else {
|
||||
panic!("native seed is valid");
|
||||
};
|
||||
record.replica_schema_version = 1;
|
||||
for member in &mut record.committed.as_mut().expect("seed commit").replicas {
|
||||
member.set_id = None;
|
||||
}
|
||||
record.validate().expect("previous schema fixture");
|
||||
replace_native_pause_backlog_replica(&store, id, *record).await;
|
||||
}
|
||||
let fault =
|
||||
NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&store.pools[1].disk_set[0]), "publish", 1);
|
||||
assert!(
|
||||
ScannerPauseBacklogController::claim(Arc::clone(&store), now + 60)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(fault);
|
||||
let interrupted = load_scanner_pause_backlog(Arc::clone(&store))
|
||||
.await
|
||||
.expect("mixed legacy and bound records retain the rollback proof");
|
||||
assert_eq!(interrupted.ledger, original);
|
||||
let upgraded = ScannerPauseBacklogController::claim(Arc::clone(&store), now + 61)
|
||||
.await
|
||||
.expect("retry finishes the format upgrade");
|
||||
let expected = upgraded.loaded.ledger.clone();
|
||||
drop(upgraded);
|
||||
for pool in &store.pools {
|
||||
let (bytes, _) = native_replica_bytes(&pool.disk_set[0]).await;
|
||||
let ScannerPauseBacklogReplicaState::Valid(record) = decode_scanner_pause_backlog_ledger(&bytes) else {
|
||||
panic!("upgraded native record is valid");
|
||||
};
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
assert!(
|
||||
record
|
||||
.committed
|
||||
.expect("upgraded membership")
|
||||
.replicas
|
||||
.iter()
|
||||
.all(|id| id.set_id.is_some())
|
||||
);
|
||||
}
|
||||
let surviving_id = native_replica_id(&store, 2, 0).set_id;
|
||||
let (source, _) = native_replica_bytes(&store.pools[1].disk_set[0]).await;
|
||||
store
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(1, source.len() * 2)
|
||||
.await
|
||||
.expect("start retiring the middle pool");
|
||||
store
|
||||
.retire_scanner_pause_backlog_for_test(1, 0)
|
||||
.await
|
||||
.expect("hand off the native backlog");
|
||||
store
|
||||
.complete_scanner_pause_backlog_retirement_for_test(1)
|
||||
.await
|
||||
.expect("persist the completed middle pool through the ILM receipt checks");
|
||||
shutdown_native_retirement_store(store).await;
|
||||
|
||||
let reordered = super::super::tests::setup_scanner_cycle_store_with_pool_order(
|
||||
root.path(),
|
||||
false,
|
||||
&[0, 2, 3],
|
||||
1,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(native_replica_id(&reordered, 1, 0).set_id, surviving_id);
|
||||
assert_ne!(native_replica_id(&reordered, 2, 0).set_id, surviving_id);
|
||||
let loaded = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("removing a middle pool and appending a new one preserves native authority");
|
||||
assert_eq!(loaded.ledger, expected);
|
||||
assert_eq!(loaded.persistence_state, "membership_repair_pending");
|
||||
if fail_publication {
|
||||
// Match the reported partial publication: the two existing
|
||||
// pools reject the new commit while the replacement accepts it.
|
||||
let faults = [0, 1].map(|pool| {
|
||||
NativeScannerPauseBacklogWriteFault::fail_before_write(
|
||||
Arc::clone(&reordered.pools[pool].disk_set[0]),
|
||||
"publish",
|
||||
2,
|
||||
)
|
||||
});
|
||||
let error = ScannerPauseBacklogController::claim(Arc::clone(&reordered), now + 120)
|
||||
.await
|
||||
.err()
|
||||
.expect("the incomplete publication must fail");
|
||||
assert!(error.contains("pool 0 set 0") && error.contains("pool 1 set 0"), "{error}");
|
||||
drop(faults);
|
||||
let recovered = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("partial publication after reindexing retains proven authority");
|
||||
assert_eq!(recovered.ledger, expected);
|
||||
assert!(recovered.durable);
|
||||
}
|
||||
let (source, _) = native_replica_bytes(&reordered.pools[1].disk_set[0]).await;
|
||||
reordered
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(1, source.len() * 2)
|
||||
.await
|
||||
.expect("retire the reindexed member before a scanner writer completes the membership change");
|
||||
reordered
|
||||
.retire_scanner_pause_backlog_for_test(1, 0)
|
||||
.await
|
||||
.expect("retirement uses the same bound authority");
|
||||
let surviving = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("remaining members retain the complete ledger");
|
||||
assert_eq!(surviving.ledger, expected);
|
||||
assert!(surviving.all_current_replicas_stable());
|
||||
shutdown_native_retirement_store(reordered).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_single_pool_partial_commit_converges_across_24_sets() {
|
||||
@@ -2592,9 +2814,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let stable = durable_ledger(unix_now());
|
||||
let members = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let members = native_replica_ids(&store);
|
||||
let committed = ScannerPauseBacklogCommitRecord::new(stable.clone(), members);
|
||||
let record = ScannerPauseBacklogReplicaRecord::new(Some(stable), Some(committed));
|
||||
let bytes = encode_scanner_pause_backlog_record(&record).expect("stable replica should encode");
|
||||
@@ -3021,10 +3241,7 @@ mod tests {
|
||||
let now = unix_now().saturating_add(60);
|
||||
let replacement = claim_scanner_pause_backlog_writer(&original, now).expect("next writer generation");
|
||||
let new_commit = ScannerPauseBacklogCommitRecord::new(replacement.clone(), current_ids.clone());
|
||||
let failed_id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: failed_pool,
|
||||
set_index: 0,
|
||||
};
|
||||
let failed_id = native_replica_id(&store, failed_pool, 0);
|
||||
let fault = NativeScannerPauseBacklogWriteFault::fail_before_write(
|
||||
Arc::clone(&store.pools[failed_pool].disk_set[0]),
|
||||
"publish",
|
||||
@@ -3553,10 +3770,8 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.expect("old cohort commit");
|
||||
let now = unix_now().saturating_add(60);
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let newer_member = replica_id(2, 1);
|
||||
let ids = native_replica_ids(&store);
|
||||
let newer_member = native_replica_id(&store, 2, 1);
|
||||
let newer = claim_scanner_pause_backlog_writer(&original, now).expect("newer generation");
|
||||
let newer_commit = ScannerPauseBacklogCommitRecord::new(newer.clone(), ids.clone());
|
||||
for id in &ids {
|
||||
@@ -3617,14 +3832,12 @@ mod tests {
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let wedge = native_interleaved_expansion_wedge(&store, &original, now).await;
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let ids = native_replica_ids(&store);
|
||||
// The retiring source holds the newest stable ledger.
|
||||
let newest = claim_scanner_pause_backlog_writer(&wedge.acknowledged, now + 1).expect("newest generation");
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
replica_id(0, 0),
|
||||
native_replica_id(&store, 0, 0),
|
||||
ScannerPauseBacklogReplicaRecord::new(
|
||||
Some(newest.clone()),
|
||||
Some(ScannerPauseBacklogCommitRecord::new(newest.clone(), ids)),
|
||||
@@ -3675,10 +3888,8 @@ mod tests {
|
||||
set_runtime_error(None);
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let newer_member = replica_id(2, 1);
|
||||
let ids = native_replica_ids(&store);
|
||||
let newer_member = native_replica_id(&store, 2, 1);
|
||||
let others = ids.iter().copied().filter(|id| *id != newer_member).collect::<Vec<_>>();
|
||||
let middle = claim_scanner_pause_backlog_writer(&original, now).expect("middle generation");
|
||||
let newest = claim_scanner_pause_backlog_writer(&middle, now + 1).expect("newest generation");
|
||||
@@ -3837,10 +4048,8 @@ mod tests {
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let wedge = native_interleaved_expansion_wedge(&store, &original, now).await;
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(wedge.last, replica_id(2, 1));
|
||||
let ids = native_replica_ids(&store);
|
||||
assert_eq!(wedge.last, native_replica_id(&store, 2, 1));
|
||||
|
||||
// A different stable ledger at the newest key is a fork.
|
||||
let fork = claim_scanner_pause_backlog_writer(&original, now + 7).expect("forked writer generation");
|
||||
@@ -3851,7 +4060,7 @@ mod tests {
|
||||
assert_ne!(fork, wedge.acknowledged);
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
replica_id(1, 0),
|
||||
native_replica_id(&store, 1, 0),
|
||||
ScannerPauseBacklogReplicaRecord::new(
|
||||
Some(fork.clone()),
|
||||
Some(ScannerPauseBacklogCommitRecord::new(fork, ids.clone())),
|
||||
@@ -3868,7 +4077,7 @@ mod tests {
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(native_replica_snapshot(&store).await, forked, "a refused claim keeps every record");
|
||||
replace_native_pause_backlog_replica(&store, replica_id(1, 0), wedge.rollback.clone()).await;
|
||||
replace_native_pause_backlog_replica(&store, native_replica_id(&store, 1, 0), wedge.rollback.clone()).await;
|
||||
|
||||
// A missing member that a stored commit names may have lost a
|
||||
// newer stable ledger.
|
||||
@@ -3903,7 +4112,7 @@ mod tests {
|
||||
.expect_err("retirement must refuse an unproven authority");
|
||||
assert!(refused.to_string().contains("unsafe authority conflict"), "{refused}");
|
||||
assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source_bytes);
|
||||
replace_native_pause_backlog_replica(&store, replica_id(1, 1), wedge.rollback.clone()).await;
|
||||
replace_native_pause_backlog_replica(&store, native_replica_id(&store, 1, 1), wedge.rollback.clone()).await;
|
||||
|
||||
// The same records without the conflict converge and retire.
|
||||
for set in 0..2 {
|
||||
@@ -4020,7 +4229,7 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.as_ref()
|
||||
.expect("seed retains the old cohort proof");
|
||||
assert_eq!(committed.replicas, vec![replica_id(0, 0), replica_id(0, 1)]);
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 0, 0), native_replica_id(&store, 0, 1)]);
|
||||
assert_eq!(loaded.replica_count, 6);
|
||||
if loaded.healthy_replicas == loaded.replica_count {
|
||||
break loaded;
|
||||
@@ -4037,13 +4246,13 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.as_ref()
|
||||
.expect("seed retains the old cohort proof");
|
||||
assert_eq!(committed.replicas, vec![replica_id(0, 0), replica_id(0, 1)]);
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 0, 0), native_replica_id(&store, 0, 1)]);
|
||||
assert_eq!(loaded.replica_count, 6);
|
||||
assert_eq!(loaded.healthy_replicas, 6);
|
||||
let seeded = loaded
|
||||
.replicas
|
||||
.iter()
|
||||
.find(|replica| replica.id == replica_id(2, 0))
|
||||
.find(|replica| replica.id == native_replica_id(&store, 2, 0))
|
||||
.expect("the canceled caller's admitted seed replica");
|
||||
assert!(matches!(
|
||||
&seeded.state,
|
||||
@@ -4055,7 +4264,7 @@ mod tests {
|
||||
let remaining = loaded
|
||||
.replicas
|
||||
.iter()
|
||||
.find(|replica| replica.id == replica_id(2, 1))
|
||||
.find(|replica| replica.id == native_replica_id(&store, 2, 1))
|
||||
.expect("the remaining seed replica");
|
||||
assert!(matches!(
|
||||
&remaining.state,
|
||||
@@ -4268,7 +4477,12 @@ mod tests {
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(0, source_before.len() * 2)
|
||||
.await
|
||||
.expect("retire the old cohort's first pool");
|
||||
let surviving = [replica_id(1, 0), replica_id(1, 1), replica_id(2, 0), replica_id(2, 1)];
|
||||
let surviving = [
|
||||
native_replica_id(&store, 1, 0),
|
||||
native_replica_id(&store, 1, 1),
|
||||
native_replica_id(&store, 2, 0),
|
||||
native_replica_id(&store, 2, 1),
|
||||
];
|
||||
let newer = claim_scanner_pause_backlog_writer(&old.ledger, now + 2).expect("surviving writer");
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
@@ -4333,7 +4547,7 @@ mod tests {
|
||||
assert_eq!(loaded.ledger, original);
|
||||
assert!(loaded.durable && loaded.stable_matches_ledger);
|
||||
let committed = loaded.authoritative_commit.expect("survivor commit proof");
|
||||
assert_eq!(committed.replicas, vec![replica_id(2, 0), replica_id(2, 1)]);
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 2, 0), native_replica_id(&store, 2, 1)]);
|
||||
}
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
@@ -4911,7 +5125,135 @@ mod tests {
|
||||
}
|
||||
|
||||
fn replica_id(pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId { pool_index, set_index }
|
||||
ScannerPauseBacklogReplicaId {
|
||||
pool_index,
|
||||
set_index,
|
||||
set_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn native_replica_id(store: &ECStore, pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId::from_set(&store.pools[pool_index].disk_set[set_index])
|
||||
.expect("native replica has an immutable format identity")
|
||||
}
|
||||
|
||||
fn native_replica_ids(store: &ECStore) -> Vec<ScannerPauseBacklogReplicaId> {
|
||||
store
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| pool.disk_set.iter())
|
||||
.map(|set| ScannerPauseBacklogReplicaId::from_set(set).expect("native replica format identity"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bound_replica_id(pool_index: usize, identity: u128) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId {
|
||||
pool_index,
|
||||
set_index: 0,
|
||||
set_id: Some(uuid::Uuid::from_u128(identity)),
|
||||
}
|
||||
}
|
||||
|
||||
fn missing_replica(id: ScannerPauseBacklogReplicaId) -> ScannerPauseBacklogReplica {
|
||||
ScannerPauseBacklogReplica {
|
||||
id,
|
||||
revision: Some(DataUsageCacheRevision::Missing),
|
||||
state: ScannerPauseBacklogReplicaState::Missing,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_survives_pool_reindexing() {
|
||||
let ledger = durable_ledger(50);
|
||||
let previous = bound_replica_id(2, 22);
|
||||
let current = bound_replica_id(1, 22);
|
||||
let added = bound_replica_id(2, 33);
|
||||
let record = replica_record_for_members(&ledger, &ledger, &[previous]);
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
let loaded = select_scanner_pause_backlog_replicas(vec![
|
||||
missing_replica(bound_replica_id(0, 11)),
|
||||
decoded_replica(current, &record),
|
||||
missing_replica(added),
|
||||
])
|
||||
.expect("a surviving set keeps its commit proof after its routing index changes");
|
||||
assert_eq!(loaded.ledger, ledger);
|
||||
assert!(loaded.durable);
|
||||
assert_eq!(loaded.authoritative_commit.expect("surviving native proof").replicas, vec![current]);
|
||||
|
||||
let mut replicas = vec![retirement_replica(current, &record)];
|
||||
for id in [bound_replica_id(0, 11), added] {
|
||||
replicas.push(ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
let plan = plan_scanner_pause_backlog_retirement(1, &replicas)
|
||||
.expect("reindexed native authority permits safe retirement")
|
||||
.expect("new survivors need the existing ledger");
|
||||
assert_eq!(planned_stable_ledger(&plan), ledger);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_does_not_rebind_a_missing_set_to_its_reused_index() {
|
||||
let ledger = durable_ledger(50);
|
||||
let missing = bound_replica_id(2, 22);
|
||||
let replacement = bound_replica_id(2, 33);
|
||||
let record = replica_record_for_members(&ledger, &ledger, &[missing]);
|
||||
assert!(
|
||||
select_scanner_pause_backlog_replicas(vec![
|
||||
missing_replica(bound_replica_id(0, 11)),
|
||||
decoded_replica(replacement, &record),
|
||||
])
|
||||
.is_err(),
|
||||
"copied bytes on a new set cannot impersonate the missing commit member"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_membership_binds_only_after_its_existing_proof_is_verified() {
|
||||
let ledger = durable_ledger(50);
|
||||
let legacy = replica_record_for_members(&ledger, &ledger, &[replica_id(2, 0)]);
|
||||
assert_eq!(legacy.replica_schema_version, 1);
|
||||
let current = bound_replica_id(2, 22);
|
||||
let loaded = select_scanner_pause_backlog_replicas(vec![decoded_replica(current, &legacy)])
|
||||
.expect("unchanged legacy membership remains readable");
|
||||
let upgraded = stable_scanner_pause_backlog_record(
|
||||
&loaded.ledger,
|
||||
loaded.authoritative_commit.as_ref(),
|
||||
&scanner_pause_backlog_replica_ids(&loaded.replicas),
|
||||
);
|
||||
upgraded
|
||||
.validate()
|
||||
.expect("the replacement record binds every committed member");
|
||||
assert_eq!(upgraded.replica_schema_version, 1);
|
||||
assert_eq!(upgraded.committed.expect("bound membership").replicas, vec![current]);
|
||||
|
||||
assert!(
|
||||
select_scanner_pause_backlog_replicas(vec![
|
||||
decoded_replica(bound_replica_id(1, 22), &legacy),
|
||||
missing_replica(bound_replica_id(2, 33)),
|
||||
])
|
||||
.is_err(),
|
||||
"an already stranded legacy proof must not guess its old member's identity"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_rejects_nil_duplicate_and_mixed_identities() {
|
||||
let ledger = durable_ledger(50);
|
||||
for members in [
|
||||
vec![bound_replica_id(0, 0)],
|
||||
vec![bound_replica_id(0, 11), bound_replica_id(1, 11)],
|
||||
vec![bound_replica_id(0, 11), bound_replica_id(0, 22)],
|
||||
vec![bound_replica_id(0, 11), replica_id(1, 0)],
|
||||
] {
|
||||
let record = replica_record_for_members(&ledger, &ledger, &members);
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
assert!(record.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
fn decoded_replica(
|
||||
@@ -5155,6 +5497,7 @@ mod tests {
|
||||
.map(|set_index| ScannerPauseBacklogReplicaId {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
set_id: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
replica_record_for_members(stable, committed, &replicas)
|
||||
@@ -5167,6 +5510,7 @@ mod tests {
|
||||
ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: Some(serde_json::to_vec(record).expect("native record bytes")),
|
||||
}
|
||||
@@ -5294,6 +5638,7 @@ mod tests {
|
||||
replicas.extend(targets.iter().map(|id| ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: None,
|
||||
}));
|
||||
|
||||
@@ -113,17 +113,37 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_prein
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_with_pool_order(
|
||||
root,
|
||||
seed_usage_baseline,
|
||||
&(0..pool_count).collect::<Vec<_>>(),
|
||||
sets_per_pool,
|
||||
drives_per_set,
|
||||
preinitialize_disks,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_with_pool_order(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_order: &[usize],
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let pool_count = pool_order.len();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
for (pool_index, physical_pool) in pool_order.iter().enumerate() {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..sets_per_pool {
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = if sets_per_pool == 1 {
|
||||
root.join(format!("pool{pool_index}/disk{disk_index}"))
|
||||
root.join(format!("pool{physical_pool}/disk{disk_index}"))
|
||||
} else {
|
||||
root.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"))
|
||||
root.join(format!("pool{physical_pool}/set{set_index}/disk{disk_index}"))
|
||||
};
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
@@ -144,7 +164,7 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_prein
|
||||
cmd_line: if pool_count == 1 && sets_per_pool == 1 {
|
||||
"scanner-cycle-metrics".to_string()
|
||||
} else {
|
||||
format!("scanner-cycle-metrics-pool-{pool_index}")
|
||||
format!("scanner-cycle-metrics-pool-{physical_pool}")
|
||||
},
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
});
|
||||
|
||||
@@ -2348,6 +2348,10 @@ impl FolderScanner {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Budget and admission waits already pace this walker. Charge
|
||||
// only work performed after admission, never those waits again.
|
||||
let mut work_elapsed = work_started.elapsed();
|
||||
|
||||
// Reuse a resolved symlink's size; otherwise admit the size
|
||||
// stat only when the byte budget needs it.
|
||||
let budgeted_bytes = if self.sleeper.budgets_disk_io_bytes() {
|
||||
@@ -2357,7 +2361,12 @@ impl FolderScanner {
|
||||
wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
|
||||
wait_for_scanner_io(
|
||||
&ctx,
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, native_io::metadata(&item.path)),
|
||||
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, async {
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = native_io::metadata(&item.path).await;
|
||||
work_elapsed = work_elapsed.saturating_add(started.elapsed());
|
||||
result
|
||||
}),
|
||||
)
|
||||
.await??
|
||||
.map(|metadata| metadata.len())
|
||||
@@ -2369,6 +2378,7 @@ impl FolderScanner {
|
||||
wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, budgeted_bytes)).await?;
|
||||
|
||||
let size_result = run_scanner_metadata_operation(self.node_admission.as_ref(), &ctx, async {
|
||||
let started = tokio::time::Instant::now();
|
||||
self.object_metadata_visits.increment(1);
|
||||
#[cfg(feature = "diagnostic-prefix-attribution")]
|
||||
if let Some(attribution) = &self.prefix_attribution {
|
||||
@@ -2384,9 +2394,11 @@ impl FolderScanner {
|
||||
{
|
||||
attribution.record_success(&item.bucket, &item.prefix);
|
||||
}
|
||||
work_elapsed = work_elapsed.saturating_add(started.elapsed());
|
||||
result
|
||||
})
|
||||
.await?;
|
||||
let work_started = tokio::time::Instant::now();
|
||||
if self.node_admission.is_some() && ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
@@ -2483,8 +2495,12 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
wait_for_scanner_io(&ctx, self.sleeper.pace_object(&mut self.pacing_debt, work_started.elapsed()))
|
||||
.await?;
|
||||
wait_for_scanner_io(
|
||||
&ctx,
|
||||
self.sleeper
|
||||
.pace_object(&mut self.pacing_debt, work_elapsed.saturating_add(work_started.elapsed())),
|
||||
)
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -2507,7 +2523,12 @@ impl FolderScanner {
|
||||
self.checkpoint_objects = self.checkpoint_objects.saturating_add(1);
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
|
||||
wait_for_scanner_io(&ctx, self.sleeper.pace_object(&mut self.pacing_debt, work_started.elapsed())).await?;
|
||||
wait_for_scanner_io(
|
||||
&ctx,
|
||||
self.sleeper
|
||||
.pace_object(&mut self.pacing_debt, work_elapsed.saturating_add(work_started.elapsed())),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
|
||||
@@ -29,6 +29,69 @@ impl Drop for IdleGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn object_pacing_does_not_multiply_disk_budget_waits() {
|
||||
let _idle = IdleGuard::enabled();
|
||||
for corrupt in [false, true] {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
scanner.old_cache.info.name = "bucket".to_owned();
|
||||
scanner.new_cache.info.name = "bucket".to_owned();
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Default).for_disk_with_budget_for_tests(
|
||||
&temp_dir,
|
||||
crate::sleeper::ScannerDiskIoBudgetLimits {
|
||||
iops: Some(0),
|
||||
bytes_per_sec: Some(0),
|
||||
},
|
||||
);
|
||||
let mut metadata = FileMeta::new();
|
||||
let mut info = FileInfo::new("object", 4, 2);
|
||||
info.volume = "bucket".to_owned();
|
||||
info.size = 17;
|
||||
info.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture timestamp"));
|
||||
metadata.add_version(info).expect("add object metadata");
|
||||
let bytes = if corrupt {
|
||||
Vec::new()
|
||||
} else {
|
||||
metadata.marshal_msg().expect("encode object metadata")
|
||||
};
|
||||
write_test_object_metadata_bytes(&temp_dir, "bucket", "object", &bytes).await;
|
||||
let mut entry = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_owned(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut entry,
|
||||
)
|
||||
.await
|
||||
.expect("finish budget-limited folder scan");
|
||||
let total = scanner
|
||||
.new_cache
|
||||
.size_recursive(&hash_path("bucket").0)
|
||||
.expect("root accounting");
|
||||
assert_eq!(total.objects, usize::from(!corrupt));
|
||||
assert_eq!(total.size, if corrupt { 0 } else { 17 });
|
||||
assert_eq!(total.failed_objects, usize::from(corrupt));
|
||||
let budget = crate::sleeper::scanner_disk_io_budget_status()
|
||||
.into_iter()
|
||||
.find(|status| status.disk == temp_dir.to_string_lossy())
|
||||
.expect("disk budget status");
|
||||
assert!(budget.wait_seconds >= 2.0, "the scan must actually exhaust its disk budget");
|
||||
assert_eq!(scanner.pacing_debt.charged_objects, 1);
|
||||
assert!(
|
||||
scanner.pacing_debt.charged_delay < Duration::from_millis(100),
|
||||
"budget waits must not become proportional work debt: {:?}",
|
||||
scanner.pacing_debt.charged_delay
|
||||
);
|
||||
assert_eq!(scanner.pacing_debt.pending(), Duration::ZERO);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn pacing_preserves_real_folder_versions_bytes_checkpoints_and_maintenance() {
|
||||
|
||||
@@ -136,6 +136,8 @@ pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
|
||||
/// `elapsed_work_time * factor`, clamped to `[MIN_SLEEP, max_sleep]`.
|
||||
/// Folder walkers accumulate proportional object waits without the per-object
|
||||
/// floor, paying bounded batches and the final bucket tail.
|
||||
/// Their work measurement excludes disk-budget and node-admission waits so
|
||||
/// foreground protection does not become an additional proportional delay.
|
||||
/// For folder-level gaps, [`sleep_folder`] uses `MIN_SLEEP * factor`,
|
||||
/// clamped only by `max_sleep`. All sleeps are gated on [`SCANNER_IDLE_MODE`];
|
||||
/// the per-disk background I/O budget bound with [`DynamicSleeper::for_disk`]
|
||||
@@ -228,6 +230,17 @@ impl DynamicSleeper {
|
||||
SCANNER_IDLE_MODE.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn foreground_backoff(&self) -> Duration {
|
||||
// Local walkers already use disk-priority admission and a shared I/O
|
||||
// budget. A global per-operation floor duplicates that control and
|
||||
// throttles idle disks because unrelated foreground work is active.
|
||||
if self.disk_io_budget.is_some() {
|
||||
Duration::ZERO
|
||||
} else {
|
||||
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity())
|
||||
}
|
||||
}
|
||||
|
||||
fn read_params(&self) -> (f64, Duration) {
|
||||
let config = self.read_config();
|
||||
(config.factor, config.max_sleep)
|
||||
@@ -250,15 +263,14 @@ impl DynamicSleeper {
|
||||
}
|
||||
let (factor, max_sleep) = self.read_params();
|
||||
if factor == 0.0 || max_sleep.is_zero() {
|
||||
let foreground_sleep =
|
||||
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let foreground_sleep = self.foreground_backoff();
|
||||
if !foreground_sleep.is_zero() {
|
||||
tokio::time::sleep(foreground_sleep).await;
|
||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let foreground_sleep = self.foreground_backoff();
|
||||
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
|
||||
.min(max_sleep)
|
||||
.max(foreground_sleep);
|
||||
@@ -352,8 +364,7 @@ impl SleepTimer {
|
||||
}
|
||||
let (factor, max_sleep) = self.sleeper.read_params();
|
||||
if factor == 0.0 || max_sleep.is_zero() {
|
||||
let foreground_sleep =
|
||||
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let foreground_sleep = self.sleeper.foreground_backoff();
|
||||
if !foreground_sleep.is_zero() {
|
||||
tokio::time::sleep(foreground_sleep).await;
|
||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||
@@ -361,7 +372,7 @@ impl SleepTimer {
|
||||
return;
|
||||
}
|
||||
let elapsed = self.start.elapsed();
|
||||
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let foreground_sleep = self.sleeper.foreground_backoff();
|
||||
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
|
||||
.max(MIN_SLEEP)
|
||||
.min(max_sleep)
|
||||
|
||||
@@ -43,6 +43,8 @@ pub(crate) struct PacingDebt {
|
||||
pub(crate) charged_objects: u64,
|
||||
#[cfg(test)]
|
||||
pub(crate) batches: u64,
|
||||
#[cfg(test)]
|
||||
pub(crate) charged_delay: Duration,
|
||||
}
|
||||
|
||||
impl PacingDebt {
|
||||
@@ -73,6 +75,7 @@ impl PacingDebt {
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.charged_objects += 1;
|
||||
self.charged_delay = self.charged_delay.saturating_add(delay);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,7 +135,7 @@ impl DynamicSleeper {
|
||||
if let Some(elapsed) = elapsed {
|
||||
debt.charge(proportional_delay(elapsed, config), Instant::now());
|
||||
}
|
||||
let foreground = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let foreground = self.foreground_backoff();
|
||||
let now = Instant::now();
|
||||
let folder_reaches_deadline = folder
|
||||
&& debt.started.is_some_and(|start| {
|
||||
|
||||
@@ -181,6 +181,54 @@ async fn active_wait_wakes_for_fastest_idle_off_and_smaller_max_wait() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn disk_bound_pacing_does_not_repeat_global_foreground_backoff() {
|
||||
let _idle = IdleGuard::enabled();
|
||||
struct ForegroundGuard;
|
||||
impl Drop for ForegroundGuard {
|
||||
fn drop(&mut self) {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
}
|
||||
}
|
||||
let _foreground = ForegroundGuard;
|
||||
crate::set_foreground_read_activity(25);
|
||||
let disk = tempfile::tempdir().expect("disk identity");
|
||||
let sleeper = DynamicSleeper::new(ScannerSpeed::Default).for_disk_with_budget_for_tests(
|
||||
disk.path(),
|
||||
ScannerDiskIoBudgetLimits {
|
||||
iops: Some(0),
|
||||
bytes_per_sec: None,
|
||||
},
|
||||
);
|
||||
let mut debt = PacingDebt::default();
|
||||
let begin = Instant::now();
|
||||
for _ in 0..100 {
|
||||
sleeper.pace_object(&mut debt, Duration::from_micros(20)).await;
|
||||
}
|
||||
sleeper.flush_pacing(&mut debt).await;
|
||||
assert_eq!(begin.elapsed(), Duration::from_millis(4));
|
||||
assert_eq!(debt.charged_objects, 100);
|
||||
assert_eq!(debt.pending, Duration::ZERO);
|
||||
// The first operation consumes the token bucket's one-operation burst.
|
||||
sleeper.admit_disk_io(1, 0).await;
|
||||
let begin = Instant::now();
|
||||
sleeper.sleep_folder_paced(&mut debt).await;
|
||||
assert_eq!(
|
||||
begin.elapsed(),
|
||||
SCANNER_DISK_IO_BUDGET_MAX_WAIT + Duration::from_millis(2),
|
||||
"the per-disk budget and preset folder gap must still apply"
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_disk_io_budget_status()
|
||||
.into_iter()
|
||||
.find(|status| status.disk == disk.path().to_string_lossy())
|
||||
.expect("disk charges remain visible")
|
||||
.charged_ops,
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn foreground_backoff_flushes_tail_without_double_charging_or_max_wait_cap() {
|
||||
|
||||
@@ -591,6 +591,34 @@ pub trait AuthoritativeListOperations: ListOperations {
|
||||
max_keys: i32,
|
||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error>;
|
||||
|
||||
/// Enumerate an exact dotted leaf below identifier-like directory components
|
||||
/// containing ASCII letters, digits, `_` or `-`. Noncandidate object metadata
|
||||
/// is outside the scan. Namespace components may occur at any depth; reserved
|
||||
/// words do not identify a subtree that can be skipped.
|
||||
///
|
||||
/// Returned keys retain the quorum, unversioned, EOF and cancellation
|
||||
/// guarantees of `list_unversioned_objects_authoritative`. Older peers may
|
||||
/// perform extra I/O; the coordinator must filter their extra rows before
|
||||
/// resolving metadata. Callers fence concurrent mutations separately.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects an invalid directory prefix/leaf, invalid page size, uncertain
|
||||
/// matching metadata, versioned entries or insufficient read quorum. Backends
|
||||
/// without this capability return Unsupported; ordinary listing is not a fallback.
|
||||
async fn list_identifier_tree_leaves_authoritative(
|
||||
self: Arc<Self>,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_leaf: &str,
|
||||
_marker: Option<String>,
|
||||
_max_keys: i32,
|
||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error>
|
||||
where
|
||||
Self::Error: From<std::io::Error>,
|
||||
{
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "authoritative identifier-tree listing is not supported").into())
|
||||
}
|
||||
|
||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||
///
|
||||
/// # Errors
|
||||
|
||||
@@ -19,6 +19,8 @@ collection:
|
||||
|
||||
- [Table maintenance periodic queueing](operations/s3-tables-maintenance-scheduler.md) — opt-in scheduling, limits, worker handoff, and recovery.
|
||||
|
||||
- [S3 Tables catalog lifecycle](operations/s3-tables-catalog-lifecycle.md) — disablement, fenced empty-table cleanup, and registration-cache boundaries.
|
||||
|
||||
For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity,
|
||||
see [PUT cleanup isolation](operations/put-cleanup-isolation.md).
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
|
||||
| Document | Use this when |
|
||||
|---|---|
|
||||
| [s3-compatibility-matrix.md](s3-compatibility-matrix.md) | writing or checking a user-facing S3 compatibility claim, or moving a Ceph s3tests case between lists |
|
||||
| [s3-tables-support-matrix.md](s3-tables-support-matrix.md) | writing a release note or client-compatibility statement about S3 Tables / Iceberg REST Catalog (cutover procedure: [../operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md)) |
|
||||
| [s3-tables-support-matrix.md](s3-tables-support-matrix.md) | writing a release note or client-compatibility statement about S3 Tables / Iceberg REST Catalog |
|
||||
| [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md) | a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different |
|
||||
| [minio-file-format-compat.md](minio-file-format-compat.md) | deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching a listed version anchor |
|
||||
|
||||
|
||||
@@ -35,9 +35,6 @@
|
||||
- `s3gate-parse-strict` unreadable bucket versioning on object writes: a stored versioning configuration that cannot be parsed was historically read as unversioned, and existing clusters may already hold such buckets unnoticed. RUSTFS_BUCKET_CONFIG_PARSE_MODE defaults to permissive, which keeps that unversioned write and records it through rustfs_bucket_metadata_parse_failed_total, rustfs_bucket_metadata_unparsable_current, and an error log; strict refuses the write with 503. Paths that already refuse an unreadable configuration (typed getters, the read-modify-write guard, Object Lock, default encryption, delete-time versioning) refuse in both modes. Mixed-version risk window: older nodes lack the per-config read-modify-write guard and still write unversioned, so the metric must be driven to zero before relying on strict. Flip the default to strict after the parse-failure counter has read zero fleet-wide for two consecutive releases, and remove the permissive branch and the switch one release after that.
|
||||
- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
|
||||
- `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
|
||||
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
|
||||
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
|
||||
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
|
||||
- `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus.
|
||||
- `peer-drive-admission-v1` startup peer re-admission capability: a restarting node treats an unimplemented PeerDriveAdmission RPC as a peer that cannot report, so the startup re-admission gate fails open during rolling upgrades from older releases. Remove the fallback after the minimum supported RustFS peer version implements PeerDriveAdmission.
|
||||
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
|
||||
|
||||
@@ -72,6 +72,8 @@ Ordinary publication rechecks the selected pool against the durable pool metadat
|
||||
|
||||
Running, queued, failed, canceled, and completed decommission states exclude the source from new ordinary publication, including new multipart uploads. Previously created multipart uploads retain their drain path while the source remains non-terminal; terminal source states reject further multipart publication. Failed and canceled entries become writable for new ordinary publication only after an allowed clear operation removes that state. This check does not change repair admission or the separate fence for operations that only release capacity.
|
||||
|
||||
Bucket creation and deletion remove stale bucket usage through namespace maintenance, independently of the scanner publication pause. Failed or canceled decommission state alone does not reject that cleanup. The update checks the logical snapshot revision while holding the fixed and physical object locks, then conditionally publishes to a writable pool; a suspended source remains available for decommission to reconcile against the newer target. Cleanup preserves the scanner epoch and cycle, and retains the bucket namespace and movement guards until its physical writes finish even if the request is canceled. Namespace lease loss still fences the write. Scanner publication continues to require its existing movement admission.
|
||||
|
||||
For mixed batch deletes, only the pools selected to receive new delete markers are publication targets. Exact-version deletions on other pools remain protected by the same pool metadata read fence, without treating the retiring source or an unrelated reserved target as a destination for those markers.
|
||||
|
||||
### Shared Capacity On Healthy Targets
|
||||
@@ -143,6 +145,10 @@ Existing capacity-owner and mutation checks reconcile the pending intent before
|
||||
source cleanup; the replica exception does not clear an unknown intent, rewrite
|
||||
the native target, or change the scanner's committed-membership selection.
|
||||
|
||||
Backlog membership also carries an optional `set_id`, the first disk-slot UUID from the erasure set's persisted format. This identity survives pool removal, reordering, and disk replacement; `pool_index` and `set_index` remain routing locations. Readers resolve bound members by that UUID before checking the existing commit and rollback proofs. A new pool at a removed member's old index cannot impersonate that member. New publications bind every member, while positional records remain readable and gain identities only after their existing authority is verified. Replica schema 1 is retained; 1.0.4 readers ignore the additive identity field and still use positional membership.
|
||||
|
||||
Finish upgrading all scanner-capable nodes before changing pool membership. An older writer can replace a bound record with a positional one. If a legacy record was already stranded by pool reindexing, the missing identity cannot be guessed safely: restore the known topology in which its proof was readable, then let the updated writer bind the membership before changing it again. Do not delete or reset the backlog ledger to bypass this check. Partial writes, canceled callers, missing members, and conflicting authority remain subject to the native CAS and recovery rules above.
|
||||
|
||||
## MinIO Divergence Decisions
|
||||
|
||||
Behavior that is close to MinIO but not byte-for-byte identical. Changing either decision requires an operator compatibility note and updated characterization tests.
|
||||
|
||||
@@ -45,18 +45,9 @@ In a distributed deployment, startup publishes `FullReady` only after that local
|
||||
- Liveness reports process availability and must not depend on storage, IAM,
|
||||
lock quorum, or peer health.
|
||||
- Node readiness (`/health/ready`, `/minio/health/ready`) reports whether this process can serve reads: storage read quorum, IAM, and shared-lock quorum. It is the Kubernetes Service membership signal. Losing write quorum, the pool-metadata writer, or exclusive-lock quorum does not remove a node that can still read. The public health handler still returns 503 until startup publishes `FullReady`, so a process that has never opened S3 admission does not join the Service. Runtime storage diagnostics do not change that startup gate.
|
||||
- A blocked pool metadata writer degrades cluster-write readiness with
|
||||
`pool_meta_write_blocked`. It does not withdraw `/health/ready`, and the node
|
||||
probe omits that reason from `degradedReasons`. Metadata save-gate inspection
|
||||
is bounded to 100 ms; contention reports `pool_metadata_check_timeout` on the
|
||||
cluster-write probe without installing a block. The node probe keeps the last
|
||||
confirmed writable observation for `details.poolMetadata` during a transient
|
||||
inspection timeout, while preserving a confirmed block and failing that
|
||||
detail closed when no fresh observation exists.
|
||||
- The authenticated cluster snapshot extends its existing node-local metadata
|
||||
gate inspection with safe reason, failure phase, and original block time. It
|
||||
distinguishes timeout from a block and changes no admission or recovery
|
||||
decision. Runtime readiness and gate status are separate bounded observations.
|
||||
- Immutable pool metadata admission proofs share the local authority gate as readers. Publication, recovery, identity changes and operations requiring serialization retain exclusive ownership and their existing distributed namespace fences. A multi-phase operation releases its reader before acquiring the writer and revalidates under exclusive ownership; it never upgrades a held reader. The gate is fair, so a writer queued behind a reader that is waiting on namespace I/O also delays later admissions until that reader finishes. Every writer publishes the authority it leaves before releasing exclusive ownership. Health inspection reads that published authority without acquiring the gate, so neither a held nor a queued writer delays it. A writer's changes become visible when it releases the gate; a latch recorded by a transaction outcome after its writer released is shared with the published authority and visible immediately. Mutating paths still check authority under the gate.
|
||||
- A blocked pool metadata writer degrades cluster-write readiness with `pool_meta_write_blocked`. It does not withdraw `/health/ready`, and the node probe omits that reason from `degradedReasons`. Metadata authority inspection reads the state published by the last local writer and does not wait for the save gate, so a held or queued metadata writer does not degrade either probe; a block that writer installs appears when it releases the gate.
|
||||
- The authenticated cluster snapshot extends its existing node-local metadata gate inspection with safe reason, failure phase, and original block time. It changes no admission or recovery decision. Runtime readiness and gate status are separate observations.
|
||||
- Cluster write readiness (`/minio/health/cluster`) requires storage write quorum, the pool-metadata write gate, and exclusive-lock quorum. Use it when a caller must know that new writes can commit. It is not the Service membership probe.
|
||||
- Cluster read readiness uses the storage read-quorum path and cluster-health timeout behavior. Its lock dependency still uses the per-set majority/write-lock health check. Actual shared namespace locks require `n - n/2`, so the cluster read probe is conservative: a four-client set can still admit reads with two clients while that probe returns 503. Node `/health/ready` uses the shared-lock threshold instead, because that is what object reads acquire.
|
||||
- `HEAD` health probes keep header/status semantics and do not require response
|
||||
@@ -91,7 +82,7 @@ For a healthy IAM and metadata writer in a four-node, one-drive-per-node EC 2+2
|
||||
|
||||
With two of four nodes up, GET, HEAD, and List of a bucket that this process has not yet cached still succeed. PUT and other mutations fail because write quorum and exclusive locks are gone.
|
||||
|
||||
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection retains its own 100 ms budget; a timeout is counted by `rustfs_pool_metadata_check_timeouts_total` and uses the last confirmed node-local gate state only within the same cache TTL. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
||||
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection reads the authority published by the last local writer and does not wait for the save gate. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
||||
|
||||
The new fields are additive. Their absence in an older response is not evidence of storage quorum. Minimal responses still contain only the existing top-level fields, liveness remains dependency-independent, and HEAD responses remain bodyless.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# S3 Tables Support Matrix
|
||||
|
||||
**Use this when:** writing a release note, README claim, or client-compatibility statement about RustFS S3 Tables / Iceberg REST Catalog, or deciding whether a feature is supported, preview, or not claimed.
|
||||
**Source of truth:** the table-catalog handlers under `rustfs/src/admin/handlers/table_catalog/`; the conformance scripts and their README in `scripts/table-catalog/`; the durable-backing cutover procedure in [docs/operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md).
|
||||
**Source of truth:** the table-catalog handlers under `rustfs/src/admin/handlers/table_catalog/`; the conformance scripts and their README in `scripts/table-catalog/`; and the object catalog backup/restore contracts exercised by the focused server tests.
|
||||
|
||||
RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on top of the RustFS S3 data plane. It does not claim parity with the AWS S3 Tables control-plane API or with vendor-specific Iceberg catalog extensions.
|
||||
|
||||
@@ -26,7 +26,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias; smoke profile defaults to signing name `s3tables`. |
|
||||
| S3 object data plane | Supported | Data, metadata, manifest, and delete files are ordinary S3 objects with table-aware policy checks on warehouse paths. |
|
||||
| S3 table-bucket listings | Supported | `ListObjects`, `ListObjectsV2`, `ListObjectVersions`, metadata-list extensions, and multipart-upload listings omit warehouse objects unless the caller has `admin:GetTableMetadata` for that table, and always omit internal `.rustfs-table` keys. Protected cursors are encrypted with a key derived from the cluster root credentials, so rotating those credentials invalidates cursors already issued. |
|
||||
| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index by listing the catalog prefix. Only table entries count toward the 4,096-object limit, so maintenance reports and other per-table objects cannot exhaust it; the listing itself is bounded to 64 pages plus one page per table entry already listed, and after each page the cursor skips the rest of the maintenance subtree the page ended in, so a report flood under one table costs a bounded number of pages. Larger catalogs fail closed with a retryable service error and require operator backfill. An optional table-entry directory (see Maintenance Matrix) answers these lookups without the prefix listing. |
|
||||
| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index using authoritative table-entry pages. The scan selects `table-entry.json` below identifier-like directory components at any depth; namespace components named `tables` or `maintenance` remain valid. Noncandidate report names are filtered in directory batches before inventory retention and metadata reads, so report history does not consume ownership pages. Selected entries retain GET-equivalent quorum and corruption checks. More than 4,096 table entries fails closed with a retryable service error and requires operator backfill. An optional table-entry directory (see Maintenance Matrix) answers these lookups without the prefix listing. |
|
||||
| Table bucket enablement | Supported | A regular bucket is enabled for catalog use and addressed as the REST catalog warehouse. |
|
||||
| Catalog-vended table credentials | Automated when enabled | Disabled by default. LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. |
|
||||
| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and warehouse ARN shape for migration docs. API parity not claimed. |
|
||||
@@ -38,7 +38,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
|
||||
| Client or engine | Status | Claim |
|
||||
|---|---|---|
|
||||
| PyIceberg | Automated | Namespace and table create, append, exact-row reload/scan, metadata-location, refs, views, namespace properties, rename identity and credential negotiation. The smoke discovers the server backing and checks maintenance configuration, scheduler/worker dry-run, diagnostics and export on both backings. Direct metadata maintenance is checked as unsupported on durable-strong. Optional vended credentials include an exact-prefix data-plane scope check. |
|
||||
| PyIceberg | Automated | Namespace and table create, append, exact-row reload/scan, metadata-location, refs, views, namespace properties, rename identity, maintenance, diagnostics, export, and credential negotiation against the object catalog. Optional vended credentials include an exact-prefix data-plane scope check. |
|
||||
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, atomic CTAS SQL through staged create, expected `row_count=2`, and a CI opt-in gate for create/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
|
||||
| Trino Iceberg REST catalog | Manual/live harness | Catalog properties and a read-only `SELECT COUNT(*)` against a PyIceberg- or Spark-created table. Write compatibility not claimed. |
|
||||
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Purge-on-drop and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES`, staged-create client integration, and vended-credential integration are not claimed. |
|
||||
@@ -51,7 +51,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| Area | Status | Claim |
|
||||
|---|---|---|
|
||||
| Live conformance evidence | Automated for PyIceberg and DuckDB | `engine_compatibility.py --print-live-evidence-schema` defines the evidence schema and promotion boundaries; both smoke scripts write validated evidence records via `--live-evidence-output`. |
|
||||
| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records commands, evidence, pass criteria, and fail-closed signals for conformance, cutover, maintenance, recovery, permissions, credential vending, and claim governance. |
|
||||
| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records commands, evidence, pass criteria, and fail-closed signals for conformance, object catalog backup/restore, maintenance, recovery, permissions, credential vending, and claim governance. |
|
||||
| Vendor compatibility gap audit | Generated harness | `engine_compatibility.py --print-vendor-audit` records provider URLs, path and warehouse shapes, auth models, validation categories, and not-claimed boundaries for the four vendor profiles. |
|
||||
| Client claim promotion | Automated for scoped clients | PyIceberg and DuckDB claims are bounded by their smoke entrypoints and recorded versions; Spark needs recorded live evidence; Trino stays read-only; Snowflake and vendor profiles stay reference-only. |
|
||||
|
||||
@@ -59,8 +59,8 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
|
||||
| Area | Status | Covered behavior |
|
||||
|---|---|---|
|
||||
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, migration, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
|
||||
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. `DELETE` disables an empty catalog while preserving objects; discovery includes `disable-supported`. Namespaces, retained resources, staged metadata, and migration fences prevent disablement. |
|
||||
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, backup/restore, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
|
||||
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. `DELETE` disables an empty catalog while preserving objects; discovery includes `disable-supported`. Namespaces, retained resources, staged metadata, and publication fences prevent disablement. |
|
||||
| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
|
||||
| Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
|
||||
| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
|
||||
@@ -68,12 +68,16 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
|
||||
| Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
|
||||
| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; vended storage sessions cannot authenticate Catalog requests; responses carry `Cache-Control: no-store, private`. |
|
||||
| Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and use zero-file retention until strong maintenance configuration is persisted. |
|
||||
| Durable catalog backup and restore | Preview / controlled | Durable-strong only. `POST /iceberg/v1/{warehouse}/catalog/backup` seals a table-bucket snapshot behind the publication and migration fences, records catalog/object watermarks, and stores an immutable backup artifact. `POST /iceberg/v1/{warehouse}/catalog/restore` verifies those watermarks, requires an explicit replacement decision for an existing target, applies the catalog snapshot with ETag-CAS, and leaves a recoverable restore intent if maintenance-state finalization fails. It does not copy data files or claim automatic cross-region failover. |
|
||||
| Catalog diagnostics and export | Supported | Object catalog state exposes recovery, consistency, commit-log, idempotency, maintenance, single-active-writer policy, and scale validation information. |
|
||||
| Catalog backup and restore | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup` creates object-catalog artifact v2 with catalog/current table metadata and an inventory of current/direct `metadata-log` history, current view metadata, manifest lists/manifests, live data/delete files, and table/partition statistics. References up to 16 MiB use size/SHA-256; larger references require the captured size/ETag. Proven superseded requests retain their original staged records and any idempotency indexes with their winning history; missing or contradictory proof blocks backup and restore. External payloads are not packaged. `POST /iceberg/v1/{warehouse}/catalog/restore` applies objects conditionally; keep the target closed to clients until restore and recovery verification finish. Per-object CAS does not establish bucket-wide atomic publication or crash isolation. |
|
||||
| Catalog backup artifact transfer | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup/export` takes strict `{"backup-id":"..."}` JSON and returns saved artifact bytes; `POST /iceberg/v1/{warehouse}/catalog/backup/import` accepts the raw artifact. Artifact transfer is limited to 64 MiB and does not publish catalog or prepare external objects. Existing object-catalog v1 bytes and their legacy catalog/current-table-metadata contract remain unchanged. |
|
||||
| Catalog backup object preparation | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup/prepare` takes strict `{"backup-id":"...","object":"...","data-base64":"..."}` with standard padded Base64, up to 16 MiB decoded and 24 MiB JSON. It conditionally creates one v2-listed object within its validated table/view scope, returning `PREPARED` or `ALREADY_PRESENT`; identical bytes are idempotent and different existing bytes are never overwritten. Preparation does not publish catalog state. |
|
||||
| Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. |
|
||||
| External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import. Vendor SDK polling and policy mirroring not claimed. |
|
||||
| Multi-table transactions | Not claimed | Single-table commit atomicity only. |
|
||||
|
||||
Backup extensions require a table-enabled warehouse and `admin:MigrateTableCatalog`; export, import, and prepare also accept `/_iceberg/v1`. Restore and disaster-recovery rollback require all writers and maintenance workers stopped and the target closed to clients until recovery verification finishes. Every serving/background node must understand persistent restore controls; otherwise keep those nodes/workers out of the target service until then. `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` confirms publication-fence deployment, not restore-control version compatibility; no restore isolation guarantee is claimed for a mixed old/new fleet.
|
||||
|
||||
## Data Plane And Credential Matrix
|
||||
|
||||
| Area | Status | Covered behavior |
|
||||
@@ -95,27 +99,30 @@ their warehouse prefixes.
|
||||
|
||||
## Maintenance Matrix
|
||||
|
||||
Object-backed and durable-strong catalogs support the configuration, scheduler,
|
||||
worker, job inspection, heartbeat, and quarantine endpoints. Durable-strong
|
||||
maintenance resolves table identity from the authoritative catalog snapshot;
|
||||
recoverable job and configuration records remain separate objects. The worker
|
||||
performs metadata and unreferenced-object cleanup without changing the catalog
|
||||
pointer. The direct `/maintenance/metadata` endpoint, including snapshot
|
||||
expiration and compaction plan/commit, remains object-backed only. A configured
|
||||
The object catalog supports the configuration, scheduler, worker, job
|
||||
inspection, heartbeat, and quarantine endpoints. Recoverable job and
|
||||
configuration records remain separate objects. The worker performs metadata and
|
||||
unreferenced-object cleanup without changing the catalog pointer. The direct
|
||||
`/maintenance/metadata` endpoint supports safe planning; snapshot expiration
|
||||
and compaction commits remain explicit controlled operations. A configured
|
||||
`background-enabled` table still requires explicit worker ticks. The standalone
|
||||
server has opt-in, allowlisted periodic queue admission; see the
|
||||
[scheduler runbook](../operations/s3-tables-maintenance-scheduler.md).
|
||||
|
||||
Renamed tables retain their original metadata paths. Scheduling, planning,
|
||||
and cleanup on both backings fail closed with a conflict if another table
|
||||
shares a scanned metadata or warehouse directory, including a recreated source
|
||||
name after rename. The object backing rechecks this ownership inventory under
|
||||
the bucket publication fence before every delete pass; the inventory reads every
|
||||
table entry in the bucket, so maintenance cost grows with the table count. Such
|
||||
tables require disjoint cleanup roots before maintenance can run; enabling
|
||||
deletion does not override this ownership check.
|
||||
Renamed tables retain their original metadata paths. Object catalog scheduling,
|
||||
planning, and cleanup fail closed with a conflict if another table shares a scanned
|
||||
metadata or warehouse directory, including a recreated source name after rename.
|
||||
The object backing rechecks this ownership inventory under the bucket publication
|
||||
fence before every delete pass; the inventory reads every table entry in the bucket,
|
||||
so maintenance cost grows with the table count. Such tables require disjoint cleanup
|
||||
roots before maintenance can run; enabling deletion does not override this ownership
|
||||
check.
|
||||
|
||||
Object-backed create, register, and commit publication share one preparation deadline, `RUSTFS_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS` (default 30, never below the object lock-acquire timeout, capped at 86,400). Migration and bucket-lock queueing, reference HEAD validation, ownership inventories, metadata parsing, and snapshot preparation consume that same interval; repeated preparation does not renew it. A create writes its initial metadata file before registration without ending the deadline; when registration then fails before any catalog write may have begun, including on expiry, the handler removes that file, so the worst outcome of an expired create is a small orphan under a root no table owns, left by a crash or failed delete between the write and the removal. Expiry before a possible durable catalog write returns HTTP 503 and releases the preparation locks; a valid large operation that cannot finish within the interval keeps failing on retry until the interval is raised. Once a catalog write may have begun, this deadline no longer cancels that transaction. Individual PUT completion and persisted commit/rename recovery retain their existing cancellation guarantees. This is a preparation limit, not a hard bound on disk I/O or the durable tail. The durable-strong backing does not implement this deadline; it is being retired. A rename that the client abandons mid-flight completes or recovers on a detached task, so the bucket's `active_rename_id` is always cleared.
|
||||
Object catalog rename uses a persisted intent and publication fences. Recovery of an
|
||||
interrupted rename and recovery of a partially applied catalog restore are separate
|
||||
contracts; a successful rename test does not establish restore crash isolation.
|
||||
|
||||
Object-backed create, register, and commit publication share one preparation deadline, `RUSTFS_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS` (default 30, never below the object lock-acquire timeout, capped at 86,400). Migration and bucket-lock queueing, reference HEAD validation, ownership inventories, metadata parsing, and snapshot preparation consume that same interval; repeated preparation does not renew it. A create writes its initial metadata file before registration without ending the deadline; when registration then fails before any catalog write may have begun, including on expiry, the handler removes that file, so the worst outcome of an expired create is a small orphan under a root no table owns, left by a crash or failed delete between the write and the removal. Expiry before a possible durable catalog write returns HTTP 503 and releases the preparation locks; a valid large operation that cannot finish within the interval keeps failing on retry until the interval is raised. Once a catalog write may have begun, this deadline no longer cancels that transaction. Individual PUT completion and persisted commit/rename recovery retain their existing cancellation guarantees. This is a preparation limit, not a hard bound on disk I/O or the durable tail. A rename that the client abandons mid-flight completes or recovers on a detached task, so the bucket's `active_rename_id` is always cleared.
|
||||
|
||||
Compaction commits hold inputs to a 512 MiB stored and decoded budget per rewrite group, validate the footer (including the row-group count the decoder would reserve for) and every page header before decoding a page (a recursive or over-deep Iceberg manifest Avro schema and a Parquet page that exceeds its budget are refused with HTTP 400), and run one compaction per node at a time; a concurrent request receives HTTP 503. The planner packs each rewrite group to the smaller of `max-rewrite-bytes-per-job` and that 512 MiB stored-input ceiling, so a larger configured value is accepted but yields several groups; a group whose stored inputs still exceed the ceiling at commit time (for example a plan produced by an older release) is refused with HTTP 400 naming the ceiling before any group's output is written (every group of the plan is checked first).
|
||||
|
||||
@@ -123,27 +130,11 @@ Object-backed commit publication identifies every catalog object it validated by
|
||||
|
||||
### Table-entry directory
|
||||
|
||||
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and a backing migration is refused with a conflict while the directory is active.
|
||||
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and restores require disabling the directory first. Object catalog backups exclude the derived directory state and candidates. Re-enable the directory after a restore to rebuild it from the restored entries.
|
||||
|
||||
Fallback ownership scans, including the check before metadata cleanup, use the optional `identifier_tree_leaf` walk selector. An absent selector preserves ordinary listing behavior. Older peers ignore the additive field; the coordinator filters their noncandidate rows before metadata resolution and retains read-quorum checks for every selected entry. Old peers may do more I/O or fail on unrelated corrupt metadata, so a mixed-version scan can refuse work until those peers are upgraded. Such failures never establish owner absence. This selector changes no catalog or on-disk format and requires no fleet-confirmation flag.
|
||||
|
||||
|
||||
Durable-strong rename preserves maintenance configuration, job identifiers,
|
||||
current/latest aliases, history, audit events, and quarantine state. A persisted
|
||||
rename intent protects sidecar migration across interruption: the source remains
|
||||
authoritative until the catalog CAS succeeds, and the destination records are
|
||||
ready before publication. Retry the same rename to finish pending recovery.
|
||||
Backup, restore, register, drop, and subsequent rename operations recover a
|
||||
pending migration before proceeding. Complete pending rename recovery before
|
||||
rolling back to a version that does not understand maintenance migration intents.
|
||||
|
||||
Durable-strong scheduler reports read an observational history set, not an atomic
|
||||
snapshot of all maintenance jobs. They fix at most 4096 history-object names
|
||||
under the publication read fence, scan those objects outside the fence, and
|
||||
refresh the selected 10-job audit window, effective configuration, and current
|
||||
job under a final read fence. Exceeding the history budget fails closed with an
|
||||
unavailable response; retrying does not remove that size limit. A listed object
|
||||
that disappears during the scan also returns unavailable; retry after concurrent
|
||||
rename or maintenance activity has settled. No history object is silently omitted
|
||||
when a rename removes its old path.
|
||||
|
||||
| Capability | Status | Claim |
|
||||
|---|---|---|
|
||||
@@ -164,7 +155,7 @@ when a rename removes its old path.
|
||||
| Built-in periodic scheduler | Preview / queue-only | Default off; allowlisted, paged admission with jitter, per-pass quotas, current-job checks, and cooperative drain. Workers and failed/expired-job recovery remain operator-triggered; no automatic execution or hard per-table I/O deadline. |
|
||||
| Delete-file or row-level compaction execution | Not claimed | Manual-review maintenance item. |
|
||||
|
||||
## Recovery And Strong Backing Matrix
|
||||
## Recovery And Object Backing Matrix
|
||||
|
||||
| Area | Status | Claim |
|
||||
|---|---|---|
|
||||
@@ -172,26 +163,21 @@ when a rename removes its old path.
|
||||
| Idempotent retry | Supported | Repeated commit IDs return the finalized result or surface recoverable finalization gaps. |
|
||||
| Commit publication fencing | Supported with rolling-upgrade gate | Exact object guards protect referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket fences. In scalable mode, warehouse prefixes must not overlap and enablement, first publication, drop, and relocation are serialized by the table-bucket fence. |
|
||||
| Post-CAS finalization recovery | Supported | Repairs stale or missing idempotency indexes without changing the pointer. |
|
||||
| Catalog export | Supported | Table state, commit recovery state, and backing migration information. Durable-strong export is sourced from the linearizable snapshot rather than object marker files and includes the snapshot identity needed for a later backup operation. |
|
||||
| Strong backing state transfer | Supported | Object-backed catalog state is materialized into the durable strong snapshot deterministically, ETag-CAS protected, idempotent after interrupted finalization, and validated through the restart decoder before publication; conflicts between inactive explicit namespaces and active descendants fail closed. Hydration requires a stable non-empty ETag, caps the snapshot at 64 MiB, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on first access; only object-backed migration may initialize an empty target. |
|
||||
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` (and the alias) reports inventory, recovery blockers, prefix-index readiness, identifier collisions, fence state, target agreement, and per-bucket cutover readiness. |
|
||||
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences registry changes, acquires a persistent per-bucket write fence, drains in-flight mutations, materializes the snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` cancels only while the target has not advanced. Both mutations require `admin:MigrateTableCatalog`. Procedure: [s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md). |
|
||||
| Strong snapshot rolling compatibility | Supported | Reads snapshot versions 1 and 2; writes version 1 by default and version 2 only after both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED` are set. A process rejects a lower format after observing a higher one; once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2. |
|
||||
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`; the controlled catalog backup/restore path additionally requires referenced table objects to remain available and does not provide cross-region object replication. |
|
||||
| Catalog export | Supported | Table state, current metadata pointer, commit recovery state, idempotency state, maintenance state, and object catalog manifest. |
|
||||
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`; controlled object catalog backup/restore requires referenced table objects to remain available and does not provide cross-region object replication. |
|
||||
| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal`. |
|
||||
| Durable strong snapshot backing cutover | Preview / controlled | `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. No separate external KV/WAL service is claimed; the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` labels in v1 manifests are wire-compatibility labels, not claims. |
|
||||
| Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. |
|
||||
| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple regions. |
|
||||
|
||||
## Production Failure Coverage
|
||||
|
||||
Failure probes prove RustFS does not silently advance table state on failure. Tracked cases: stale commit token or base metadata returns a conflict without advancing the pointer; missing metadata, manifest, data, or delete objects fail closed before commit or maintenance; concurrent writers produce one CAS winner and retryable conflicts; catalog and S3 permission denials prevent data-plane bypass; stale maintenance plans fail closed before deletion or commit; post-CAS finalization gaps are visible and safely recoverable; external catalog sync conflicts leave pointer, token, and generation unchanged; backing migration stays blocked until WAL and recovery replay are clean.
|
||||
Failure probes prove RustFS does not silently advance table state on failure. Tracked cases: stale commit token or base metadata returns a conflict without advancing the pointer; missing metadata, manifest, data, or delete objects fail closed before commit, backup, restore, or maintenance; concurrent writers produce one CAS winner and retryable conflicts; catalog and S3 permission denials prevent data-plane bypass; stale maintenance plans fail closed before deletion or commit; post-CAS finalization gaps are visible and safely recoverable; external catalog sync conflicts leave pointer, token, and generation unchanged; object backup/restore stays blocked until recovery and referenced-object validation are clean.
|
||||
|
||||
Do not promote a failure case from live probe or load test to an automated claim until the exact RustFS build, client version, and expected response shape are recorded.
|
||||
|
||||
## Unsupported Or Not Claimed
|
||||
|
||||
Full AWS S3 Tables control-plane parity; full MinIO AIStor private extensions; full Cloudflare R2 Data Catalog or Alibaba OSS Tables interoperability; automatic maintenance execution and failed/expired-job recovery; durable-strong snapshot-expiration and compaction plan/commit; active-active multi-region writes; multi-table transactions; no-long-term-data-credential bootstrap; online vendor SDK polling; external catalog policy mirroring; delete-file rewrite or row-level compaction execution; built-in SQL execution; Delta Lake or Hudi; end-to-end SQL row-level DML validation through Spark or Trino. DuckDB single-table UPDATE/DELETE/MERGE has its own automated smoke coverage as listed above; this does not establish other engines' DML compatibility.
|
||||
Full AWS S3 Tables control-plane parity; full MinIO AIStor private extensions; full Cloudflare R2 Data Catalog or Alibaba OSS Tables interoperability; automatic maintenance execution and failed/expired-job recovery; active-active multi-region writes; multi-table transactions; no-long-term-data-credential bootstrap; online vendor SDK polling; external catalog policy mirroring; delete-file rewrite or row-level compaction execution; built-in SQL execution; Delta Lake or Hudi; end-to-end SQL row-level DML validation through Spark or Trino. DuckDB single-table UPDATE/DELETE/MERGE has its own automated smoke coverage as listed above; this does not establish other engines' DML compatibility.
|
||||
|
||||
## Verification
|
||||
|
||||
@@ -208,6 +194,5 @@ Any stronger vendor or engine claim needs a repeatable live harness, the exact c
|
||||
## Related
|
||||
|
||||
- [Table catalog conformance scripts](../../scripts/table-catalog/README.md)
|
||||
- [Durable backing cutover runbook](../operations/s3-tables-cutover-runbook.md)
|
||||
- [Admin route action snapshot](admin-route-action-snapshot.md)
|
||||
- [Runtime capability contracts](runtime-capability-contracts.md)
|
||||
|
||||
@@ -61,7 +61,7 @@ The `object_namespace_lock` event covers store GET/read and write acquisition, s
|
||||
|
||||
The diagnostic retains the parent tracing span available at acquisition. When an HTTP request span carries `request_id`, guard events retain that parent even if the guard moves into a background PUT commit tail. Heal events retain the available task span. Missing parent fields must not be reconstructed from adjacent log lines.
|
||||
|
||||
Use `read_repair_data`, `read_repair_commit`, and `heal_object` to distinguish shared read-repair reconstruction, its exclusive revalidation/rename phase, and ordinary exclusive object healing. A PUT quorum ACK can precede the end of its rename tail and lock lifetime.
|
||||
Use `read_repair_data`, `read_repair_commit`, and `heal_object` to distinguish shared read-repair reconstruction, its exclusive revalidation/rename phase, and ordinary exclusive object healing. Application PUTs use `TailDrained` completion and await rename fan-out before returning success. Completion can succeed at write quorum with minority failures routed to repair. Internal quorum-completion callers may acknowledge before their rename tail and lock lifetime end.
|
||||
|
||||
## Distinguish the waiting chains
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ An unreadable replica, lost fence, or conditional-write conflict leaves the orig
|
||||
|
||||
- The first block emits `decommission_state` with `state=pool_metadata_blocked`, `reason`, `phase`, and `blocked_since`. A change in recovery failure classification emits `state=pool_metadata_recovery_pending`; successful recovery emits `state=pool_metadata_recovered` with the original timestamp.
|
||||
- `rustfs_pool_metadata_blocks_total{reason}` and `rustfs_pool_metadata_recoveries_total` count block and recovery transitions. The original cause and phase remain attached to local typed errors; storage/RPC error numbers and on-disk formats are unchanged.
|
||||
- Node and cluster-write readiness include `pool_meta_write_blocked`. Waiting for the metadata save mutex is bounded to 100 ms and reports `pool_metadata_check_timeout`, not a persistent block. Cluster probes retain their existing cache and overall timeout behavior. Liveness and cluster-read quorum checks are unchanged. The authenticated node-local status and safe error fields are described in [S3 write failure diagnostics](s3-write-failure-diagnostics.md).
|
||||
- Cluster-write readiness reports `pool_meta_write_blocked`; node readiness reports the writer only through `details.poolMetadata`. Readiness reads the authority published by the last local metadata writer instead of waiting for the save gate, so an active or queued metadata transaction does not degrade readiness by itself. A block installed by a writer appears when that writer releases the gate. Cluster probes retain their existing cache and overall timeout behavior. Liveness and cluster-read quorum checks are unchanged. The authenticated node-local status and safe error fields are described in [S3 write failure diagnostics](s3-write-failure-diagnostics.md).
|
||||
|
||||
Pool metadata CAS explicitly selects `TailDrainedWithOwnerRepair`: it waits for the complete rename result and returns repair responsibility to its existing deployment and transaction owner when write quorum commits but a replica fails or signatures diverge. Cache invalidation and the committed cleanup handoff precede that error. Ordinary system metadata retains quorum-success semantics under `Quorum` and `TailDrained`; draining a tail does not require every disk to succeed. A repair-required completion does not enter rollback cleanup or wait for another temporary-data deletion. The committed bytes are not rolled back. Generic MRF and heal-channel requests cannot supply the deployment and transaction authority needed to repair system metadata; pool CAS retains the uncertain outcome and its existing recovery fence. Ordinary object repair keeps its bucket-incarnation fencing.
|
||||
|
||||
|
||||
@@ -97,7 +97,7 @@ When the whole cluster (or several nodes) went down and nodes are brought back o
|
||||
|
||||
2. Logs say what the node waits for. The IAM recovery loop retries with backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable `hint` field (for example, "storage read quorum not met yet; waiting for enough cluster nodes/disks to come online"). After repeated failures the level escalates from WARN to ERROR; this still does not kill the process.
|
||||
3. Recovery is automatic. Once storage read quorum is available, pending nodes can finish IAM bootstrap on the next retry. `/health/ready` returns `200` when startup has published `FullReady` and the node can still serve reads (storage read quorum, IAM, and shared-lock quorum). Write quorum is reported separately and is required by `/minio/health/cluster`, not by Service membership. Restarting pending nodes does not speed this up.
|
||||
4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) separate `storage` / `poolMetadata` / `iam` / `lock` readiness. `storage.ready` follows read quorum (`readinessScope` is `read_quorum`); `storage.readQuorum` and `storage.writeQuorum` show the two observations. `poolMetadata.ready` is the metadata writer and does not by itself withdraw `/health/ready`. `degradedReasons` on this probe names read-path causes such as `storage_quorum_unavailable` and `storage_and_lock_unavailable`. `pool_meta_write_blocked` and `pool_metadata_check_timeout` stay on `/minio/health/cluster`. See the [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) for probe scopes and sampling limits:
|
||||
4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) separate `storage` / `poolMetadata` / `iam` / `lock` readiness. `storage.ready` follows read quorum (`readinessScope` is `read_quorum`); `storage.readQuorum` and `storage.writeQuorum` show the two observations. `poolMetadata.ready` is the metadata writer and does not by itself withdraw `/health/ready`. `degradedReasons` on this probe names read-path causes such as `storage_quorum_unavailable` and `storage_and_lock_unavailable`. `pool_meta_write_blocked` stays on `/minio/health/cluster`. See the [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) for probe scopes and sampling limits:
|
||||
|
||||
```bash
|
||||
curl -s http://<node>:9000/health/ready | jq
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# S3 Tables Catalog Lifecycle
|
||||
|
||||
**Use this when:** disabling an object-backed table catalog, cleaning up metadata left by deleted empty tables, or diagnosing catalog registration reads.
|
||||
**Source of truth:** `rustfs/src/admin/handlers/table_catalog/config.rs`, `rustfs/src/table_catalog/store/cleanup.rs`, and `rustfs/src/table_catalog/store/registration_cache.rs`.
|
||||
|
||||
## Disable A Table Catalog
|
||||
|
||||
Use `GET /iceberg/v1/buckets/{warehouse}` to inspect the table-bucket configuration. A signed `DELETE /iceberg/v1/buckets/{warehouse}` disables an empty catalog with `admin:SetTableBucket`; the `/_iceberg/v1` alias accepts the same request. Ordinary disablement and re-enablement preserve table metadata and object data.
|
||||
|
||||
Delete tables and namespaces through the catalog APIs before disabling. Active namespaces, retained catalog resources, active renames, and unfinished recovery operations block disablement. Do not manually delete internal metadata to bypass these checks. Review the bucket lifecycle configuration: disabling table protection allows existing object expiration rules to apply again.
|
||||
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; object protection remains until publication succeeds. A subsequent PUT explicitly enables the bucket again, while stale requests cannot reactivate an inactive entry. Re-enable disabled catalogs before rolling back to a version that does not recognize inactive entries.
|
||||
|
||||
## Clean Up Deleted Empty Tables
|
||||
|
||||
`DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes recognized metadata and internal records left by deleted empty tables, then disables the catalog. Ordinary table deletion continues to preserve files. Cleanup preserves the existing table-record formats and accepts eligible empty tables deleted before the server upgrade.
|
||||
|
||||
Complete the upgrade across all serving nodes and set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` before cleanup. Delete the empty tables and namespaces through catalog APIs. The cleanup principal needs `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. Omitting `cleanup` or setting it to `false` keeps ordinary disablement; repeated or invalid values are rejected.
|
||||
|
||||
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active resources, snapshots and other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete plan and all object permissions before deletion. Persisted restore barriers and legacy catalog control state also block cleanup until recovery resolves them.
|
||||
|
||||
If a storage operation fails, retry the same request. The server checks remaining objects again and retains table-bucket protection until cleanup and catalog disablement complete. It does not create a separate cleanup task file. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
|
||||
|
||||
## Registration Reads
|
||||
|
||||
Registration checks table-ID uniqueness and warehouse-directory overlap using existing catalog and warehouse-index records. Each registration obtains a complete authoritative inventory under the publication and catalog control guards. Inventory records must satisfy the same metadata commit checks as ordinary catalog reads; an incomplete or failed inventory rejects registration.
|
||||
|
||||
Each server keeps a bounded memory cache of validated ownership records. Reuse requires the same object identity and ETag in the current inventory. New or changed records are read and validated again; eviction and restarts cause additional reads. The cache is automatic, requires no configuration or new stored objects, and is isolated by application context.
|
||||
|
||||
Registration still enumerates namespace and warehouse-index prefixes, so enumeration grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A hit reuses previously validated content without reading the current body again to prove physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication; the cache does not synchronize writers that bypass those guards.
|
||||
|
||||
Existing catalog, Iceberg metadata, and Console/client request formats remain unchanged. See the [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md) for supported API boundaries.
|
||||
@@ -1,116 +0,0 @@
|
||||
# S3 Tables Durable Backing Cutover Runbook
|
||||
|
||||
**Use this when:** moving a table-catalog warehouse from object-backed catalog state to the durable strong snapshot backing (`RUSTFS_TABLE_CATALOG_BACKING=durable-strong`), or rolling the strong snapshot format from version 1 to version 2.
|
||||
**Source of truth:** the `{warehouse}/catalog/migration` routes registered in `rustfs/src/admin/handlers/table_catalog/routes.rs`; the env constants named below; claims and status labels in [docs/architecture/s3-tables-support-matrix.md](../architecture/s3-tables-support-matrix.md).
|
||||
|
||||
## Preconditions
|
||||
|
||||
| Requirement | Why |
|
||||
|---|---|
|
||||
| A principal with `GetTableCatalogAction` on each table bucket (preflight) and `admin:MigrateTableCatalog` (migration `POST` / `DELETE`). | The mutations are admin-gated. |
|
||||
| Every catalog writer runs a release that recognizes the durable-backing migration fence. | An older writer does not see the persisted fence and can mutate the object-backed source after the snapshot inventory is captured. |
|
||||
| An object-backed catalog backup, plus the current metadata pointer and version token for representative tables. | Recovery after a failed cutover is an operator-selected restore, not a restart against the stale pointer. |
|
||||
| Every mutating object-only operation (maintenance workers, catalog recovery, export, diagnostics, external catalog bridge writes) is inventoried and confirmed supported in durable-strong mode. | Unsupported operations fail closed after cutover rather than continuing against object-backed state. |
|
||||
|
||||
## Cutover Procedure
|
||||
|
||||
1. Take the object-backed backup and record pointer and version token for representative tables.
|
||||
2. Run the preflight for each warehouse and treat every `blockers` entry as fail-closed. Repair commit recovery state and backfill the warehouse prefix index before continuing. Requests are SigV4-signed with the catalog's REST signing name; the `/_iceberg/v1` alias accepts the same paths.
|
||||
|
||||
```text
|
||||
GET /iceberg/v1/{warehouse}/catalog/migration
|
||||
```
|
||||
3. Drain every catalog writer that predates the migration fence and restart it on a fence-aware release. Keep all writers on that release until cutover completes.
|
||||
4. Quiesce mutating object-only operations (step 4 of Preconditions).
|
||||
5. Run the migration `POST` with `admin:MigrateTableCatalog`. It acquires the exclusive migration fence to drain in-flight fence-aware mutations, persists the source fence while exclusivity is held, then copies catalog state and reports `ready_to_enable_durable_strong`.
|
||||
|
||||
```text
|
||||
POST /iceberg/v1/{warehouse}/catalog/migration
|
||||
```
|
||||
|
||||
6. Repeat preflight and materialization for every table bucket. Do not proceed until the preflight reports `SNAPSHOT_MATERIALIZED`, no blockers, and `ready_to_enable_durable_strong: true` for all of them.
|
||||
7. Restart with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`, then verify catalog config, table and view loads, commit idempotency, and table data-plane policy resolution before admitting writers.
|
||||
8. Preserve the object-backed backup until durable strong backing has passed the operator's retention window.
|
||||
|
||||
## Cancelling Before Cutover
|
||||
|
||||
Before the restart in step 7, `DELETE` on the migration endpoint removes a migration-created target bucket snapshot and releases the source fence. It releases the bucket fence only while the target state has not advanced, and releases the registry fence after the last bucket is cancelled. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears.
|
||||
|
||||
```text
|
||||
DELETE /iceberg/v1/{warehouse}/catalog/migration
|
||||
```
|
||||
|
||||
After the durable-strong state advances, cancellation fails closed; recovery requires an operator-selected restore or reverse migration.
|
||||
|
||||
## Controlled Catalog Backup And Restore
|
||||
|
||||
The durable catalog backup and restore endpoints are a controlled, same-table-bucket recovery mechanism. They are available only with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`; they protect the catalog snapshot and maintenance state, but do not copy or replicate table data, metadata, manifest, or delete objects.
|
||||
|
||||
Backup artifacts and restore intents use a strict internal format version. A binary rejects artifacts or pending intents from another format version; create a new backup after upgrading instead of assuming cross-version restore compatibility.
|
||||
|
||||
1. Stop or drain catalog writers and maintenance workers for the table bucket. The backup operation acquires the table-bucket publication fence and the durable migration fence, rejects pending commit recovery and pending restore recovery, rejects active maintenance leases, and records the current snapshot ETag, format version, catalog fingerprint, and referenced object watermarks.
|
||||
2. Create a backup with a principal authorized for `MigrateTableCatalogAction`. Backup creation persists an immutable recovery artifact and is therefore not covered by read-only catalog permission. The optional `expected-snapshot-etag` provides an additional compare-and-capture guard.
|
||||
|
||||
```text
|
||||
POST /iceberg/v1/{warehouse}/catalog/backup
|
||||
{"expected-snapshot-etag":"<current-etag>"}
|
||||
```
|
||||
|
||||
3. Keep every referenced table object available for the retention period. A successful backup is immutable and idempotent by content-derived backup ID; it is not a copy of those objects and cannot restore an object that was deleted or changed after capture.
|
||||
4. Before restoring, stop writers and maintenance workers again. Verify the target table bucket is still table-enabled, obtain its current snapshot ETag, and use `expected-snapshot-etag` whenever replacing an existing catalog. `allow-replace` must be explicitly `true` to replace a non-identical existing bucket snapshot.
|
||||
|
||||
```text
|
||||
POST /iceberg/v1/{warehouse}/catalog/restore
|
||||
{"backup-id":"<backup-id>","expected-snapshot-etag":"<current-etag>","allow-replace":true}
|
||||
```
|
||||
|
||||
5. Treat a restore conflict as a stale-target or changed-object condition, not as permission to retry blindly. The service verifies all recorded object watermarks before applying an ETag-CAS snapshot replacement. If the process fails after catalog replacement, retrying the same backup ID is safe only while maintenance state is exactly the persisted source state or already exactly the backup target state. A partial multi-object reconciliation fails closed with a conflict and requires operator inspection before manual cleanup.
|
||||
|
||||
This procedure is not cross-region failover. Cross-region recovery requires an independent object replication/backup system and an operator-selected catalog import or restore procedure.
|
||||
|
||||
## Strong Snapshot Version 1 to Version 2
|
||||
|
||||
1. Keep snapshot writes on version 1 during a rolling binary upgrade. Current binaries read both versions.
|
||||
2. After every catalog writer can read version 2, set both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true` and restart the catalog writers. Setting only one gate does not change the write format.
|
||||
3. Perform a controlled catalog write or migration materialization and confirm the persisted snapshot is version 2 before serving table data-plane traffic. Once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2.
|
||||
4. After any version 2 snapshot is persisted, do not roll writers back to a binary that only reads version 1. Current binaries preserve version 2 even when the gates are later disabled.
|
||||
|
||||
## Rollback And Collision Repair Rules
|
||||
|
||||
- A running process rejects restored version 1 content after observing version 2, but cannot distinguish an older snapshot with the same format version from a deliberate restore. The format high-water mark is process-local: restoring any older snapshot and restarting every writer is a privileged disaster-recovery rollback that must restore a compatible binary and an operator-selected snapshot together.
|
||||
- Migration preflight rejects an active table/view identifier collision before writing a migration fence. A pre-existing version 1 strong snapshot with such a collision loads in cleanup-only quarantine: ambiguous reads fail closed, each cleanup mutation must reduce the collision set, and unrelated writes stay blocked until all collisions are removed. Drain writers that predate cleanup quarantine before starting the repair, and finish cleanup before the first version 2 write.
|
||||
|
||||
## Related
|
||||
|
||||
- [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md)
|
||||
- [Table catalog conformance scripts](../../scripts/table-catalog/README.md) (`failure_coverage.py --print-disaster-recovery-rehearsal` generates the rehearsal for this procedure)
|
||||
|
||||
## Disable an Empty Table Catalog
|
||||
|
||||
Use `GET /iceberg/v1/buckets/{warehouse}` to check `enabled` and `disable-supported`. When supported, `DELETE` on the same route requires `admin:SetTableBucket` and returns the bucket discovery response with `enabled: false`. The `/_iceberg/v1` alias behaves identically. Existing enable and query requests remain unchanged.
|
||||
|
||||
Review the bucket lifecycle configuration before disabling: clearing table-bucket protection allows existing object expiration rules to run again. Objects are preserved by the disable operation itself. Catalog disablement requires no namespaces, retained table/view resources, or objects under `.rustfs-table/`; an active rename or backing migration also blocks it. Clean up through supported catalog operations; do not manually delete internal metadata to bypass these checks.
|
||||
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. A plain DELETE and a subsequent PUT preserve table metadata and object data.
|
||||
|
||||
### Clean Up Deleted Empty Tables
|
||||
|
||||
For the object-backed catalog, `DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes the metadata and internal records left by deleted empty tables, then disables the table catalog. Ordinary table deletion continues to preserve files. Cleanup preserves existing table record formats and can process empty tables deleted before the server upgrade.
|
||||
|
||||
Complete the server upgrade across all serving nodes and set the existing `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` setting before requesting cleanup. Delete the empty tables and their namespaces through the catalog APIs, then send the cleanup request with `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. The `/_iceberg/v1` alias accepts the same option. Omitting `cleanup` or setting it to `false` keeps the ordinary disable behavior; repeated or invalid values are rejected.
|
||||
|
||||
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active catalog resources, snapshots or other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete cleanup plan and object permissions before deleting any object. Durable-strong catalogs retain their existing disable procedure.
|
||||
|
||||
If a storage operation fails during cleanup, retry the same request. The server checks the remaining objects again and keeps the bucket's table protection enabled until cleanup and catalog disablement complete. No separate cleanup task file is created. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
|
||||
|
||||
## Object-Backed Registration Reads
|
||||
|
||||
Table registration checks table ID uniqueness and warehouse directory overlap using the existing catalog and warehouse index records. Each registration obtains a complete inventory under the existing publication and migration guards. System-bucket inventory records must satisfy the same metadata commit checks as ordinary catalog reads. An incomplete or failed inventory rejects registration.
|
||||
|
||||
Each server keeps a bounded memory cache of validated ownership records. A record can be reused only when its object identity and ETag match the current inventory. New or changed records are read and validated again. Cache eviction and process restarts cause additional reads; catalog state continues to come from the existing persisted records. The cache is automatic and requires no additional configuration or stored objects.
|
||||
|
||||
Registration still enumerates the namespace and warehouse index prefixes, so enumeration work grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A cache hit verifies the current metadata identity and reuses previously validated content; it does not re-read the current body to check its physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication.
|
||||
|
||||
Existing catalog formats and Console and Iceberg client request formats are preserved. Concurrent writers must follow the existing publication, migration, and storage fencing protocol; the memory cache does not provide synchronization for writers that bypass those guards.
|
||||
|
||||
Before rolling back to a server version without this endpoint, re-enable disabled catalogs with PUT while the supporting version is running. Older enable handlers do not reactivate inactive catalog entries. Complete the server upgrade across the cluster before using disablement.
|
||||
@@ -2,11 +2,11 @@
|
||||
|
||||
**Use this when:** planning table maintenance, enabling automatic job admission, or recovering a stalled queue.
|
||||
|
||||
The server can periodically **queue** maintenance jobs for explicitly selected table buckets. It does not run workers, delete objects, expire snapshots, or compact data automatically. Workers remain an explicit operator action. Both object-backed and durable-strong catalogs use the existing maintenance configuration and job format.
|
||||
The server can periodically **queue** maintenance jobs for explicitly selected table buckets. It does not run workers, delete objects, expire snapshots, or compact data automatically. Workers remain an explicit operator action. The object catalog uses the maintenance configuration and job format described below.
|
||||
|
||||
## Read-only Metadata Maintenance Planning
|
||||
## Direct Metadata Maintenance
|
||||
|
||||
Both backings accept authenticated planning requests at:
|
||||
The object catalog accepts authenticated maintenance requests at:
|
||||
|
||||
```text
|
||||
POST /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata
|
||||
@@ -30,13 +30,13 @@ For example:
|
||||
}
|
||||
```
|
||||
|
||||
`max-rewrite-bytes-per-job` is accepted up to any value at or above `target-file-size-bytes`, but each rewrite group is packed to at most the smaller of that value and the 512 MiB per-group stored-input ceiling that the commit path holds in memory; the 1 GiB example therefore produces groups of at most 512 MiB. The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. A nonzero retention value overrides the effective maintenance configuration for this request; omission or zero uses the effective configured retention.
|
||||
`max-rewrite-bytes-per-job` is accepted up to any value at or above `target-file-size-bytes`, but each rewrite group is packed to at most the smaller of that value and the 512 MiB per-group stored-input ceiling that the commit path holds in memory; the 1 GiB example therefore produces groups of at most 512 MiB. The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. The request's retention value overrides the configured retention for this invocation. Omission or zero adds no recent-file retention. Existing protections for referenced metadata still apply.
|
||||
|
||||
In durable-strong mode, successful planning returns HTTP 200 with a completed `DRY_RUN` job and persists its named, current, and latest reports with audit events. Read-only means that table data, metadata pointers, and catalog generations are unchanged; the request still writes maintenance report sidecars and requires the table-scoped `admin:RunTableMaintenance` permission. Inspect the returned plan and any manual-review state before taking further action.
|
||||
Successful planning returns HTTP 200 with a completed `DRY_RUN` job. It leaves table data, metadata pointers, and catalog generations unchanged. The endpoint writes named, current, and latest maintenance reports and requires the table-scoped `admin:RunTableMaintenance` permission.
|
||||
|
||||
This direct endpoint rejects `delete=true`, `commit-snapshot-expiration=true`, or `commit-compaction=true` with HTTP 400 in durable-strong mode, without publishing a job or changing table objects. The existing object-backed mutation behavior is unchanged. These restrictions do not change the separate explicit maintenance worker or quarantine APIs.
|
||||
The same endpoint supports explicit operations with `delete`, `commit-snapshot-expiration`, or `commit-compaction`. Enable at most one of these flags. Metadata deletion requires `delete-enabled`. To commit snapshot expiration or compaction, also supply the corresponding plan configuration.
|
||||
|
||||
A queued, running, or failed job, active quarantine, or inconsistent current/latest aliases or their named report returns HTTP 409 instead of replacing the existing job or discarding its retry budget. Inspect that state before retrying or taking explicit worker/recovery action. A failed publication that saved only a completed named dry-run report leaves harmless history: no alias advertises a running claim, and replanning is safe because there were no table-object effects. Planning revalidates table identity, metadata pointer, configuration, and the prior report before publication; concurrent changes also conflict. It does not introduce automatic workers, snapshot-expiration commits, or compaction commits on durable-strong backing.
|
||||
Before invoking this endpoint, including a dry run, pause scheduler admission and competing maintenance workers, then resolve outstanding jobs, active quarantine, or inconsistent current/latest aliases. The direct endpoint can replace the current and latest reports even when a job already exists; it does not guarantee HTTP 409 for these states or preserve an outstanding job's retry budget. Prior named reports remain available for inspection. If a request fails, inspect the scheduler state and named reports before retrying. Resume scheduler admission and workers only after inspecting the direct invocation's result and report state.
|
||||
|
||||
## Enablement
|
||||
|
||||
|
||||
@@ -45,15 +45,14 @@ The probe round timeout is configured independently; see [Admin peer probe timeo
|
||||
|
||||
## Correlate bounded diagnostics
|
||||
|
||||
Normal operation does not require success logs at WARN. Request counters remain available with WARN logging, while runtime readiness diagnostics distinguish `pool_meta_write_blocked`, `pool_metadata_check_timeout`, and insufficient storage quorum. Do not clear a metadata write fence merely to make readiness green.
|
||||
Normal operation does not require success logs at WARN. Request counters remain available with WARN logging, while runtime readiness diagnostics distinguish `pool_meta_write_blocked` from insufficient storage quorum. Do not clear a metadata write fence merely to make readiness green.
|
||||
|
||||
Query the authenticated `/rustfs/admin/v4/cluster/snapshot` endpoint on the affected node. `snapshot.pool_meta_write_gate` describes that node's metadata writer, not a fleet-wide aggregate. Its existing booleans are preserved; `state` and optional block details extend the same bounded, read-only gate inspection. This starts no recovery, disk reads, or additional RPCs. Runtime readiness and the gate are inspected separately, so a whole cluster snapshot is not atomic across sections.
|
||||
Query the authenticated `/rustfs/admin/v4/cluster/snapshot` endpoint on the affected node. `snapshot.pool_meta_write_gate` describes that node's metadata writer, not a fleet-wide aggregate. Its existing booleans are preserved; `state` and optional block details extend the same read-only inspection, which reads the authority published by the node's last metadata writer without waiting for the save gate. A block that an active writer installs appears when that writer releases the gate. Nodes running earlier releases can also report `check_timeout`, the `pool_metadata_check_timeout` readiness reason, and `rustfs_pool_metadata_check_timeouts_total` when their 100 ms gate inspection expired; current nodes emit none of them. This starts no recovery, disk reads, or additional RPCs. Runtime readiness and the gate are inspected separately, so a whole cluster snapshot is not atomic across sections.
|
||||
|
||||
| `state` | Meaning |
|
||||
| --- | --- |
|
||||
| `writable` | No metadata write block was observed. Other dependencies may still make the node unready. |
|
||||
| `blocked` | A metadata write block was observed; `reason`, `phase`, and `sinceUnixSecs` describe its typed failure context. |
|
||||
| `check_timeout` | The existing 100 ms inspection budget expired. Readiness remains false, but a write block is not asserted. |
|
||||
| `unavailable` | The object store or a recognized metadata observation was unavailable; no block details are invented. |
|
||||
|
||||
For `blocked`, `reason` reuses the metadata failure classification, `phase` identifies the operation stage that caused the block (not live recovery-worker progress), and `sinceUnixSecs` is the original block time in Unix seconds. Polling does not reset that time. Recovery replaces the observation with `writable` and removes the previous block details. An unavailable store reports `writesReady=false` with `state="unavailable"`. Older responses may omit `state` and block details; missing fields alone do not prove a healthy writer. Operation text, raw replica errors, disk paths, and credentials are excluded. See [Pool metadata recovery](pool-metadata-recovery.md) for recovery and escalation boundaries.
|
||||
|
||||
@@ -40,7 +40,7 @@ Bitrot cycle resolution differs because the canonical persistent key belongs to
|
||||
| `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` (`DEFAULT_SCANNER_CYCLE_MAX_OBJECTS`) | Caps objects processed by one cycle. `0` disables this budget. |
|
||||
| `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` (`DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES`) | Caps directories entered by one cycle. `0` disables this budget. |
|
||||
| `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` (`DEFAULT_HEAL_BITROT_CYCLE_SECS`, 30 days) | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
|
||||
| `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` (`DEFAULT_SCANNER_IDLE_MODE`) | Enables preset scanner sleeps and foreground-read backoff. `false` disables those sleeps; per-disk foreground-priority admission remains active. |
|
||||
| `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` (`DEFAULT_SCANNER_IDLE_MODE`) | Enables preset scanner sleeps and the foreground backoff for callers without a disk binding. `false` disables those sleeps; per-disk foreground-priority admission remains active. |
|
||||
| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `14` (`DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS`) | Timeout for saving scanner cache; runtime enforces a minimum of `1` and keeps the default persistence budget within the distributed publication lease. |
|
||||
| `scanner.recovery_backoff_cap` | `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` | seconds | `600` (`DEFAULT_SCANNER_RECOVERY_BACKOFF_CAP_SECS`) | Caps the wait between automatic startup recovery retries. Must be a positive integer; `0` and malformed values are rejected, so retries never stop. Applies to the next computed retry, not to a wait already in progress. The retry count resets after 24 hours, which bounds a single wait at about one day even with a larger cap. |
|
||||
| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` (`DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS`) | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. |
|
||||
@@ -63,7 +63,7 @@ Speed presets (`crates/config/src/constants/scanner.rs`) set the base sleep mult
|
||||
| `slow` | 10x | 15s | 60s |
|
||||
| `slowest` | 100x | 15s | 30m |
|
||||
|
||||
Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. With `idle_mode=true`, directory-level sleep is `1ms x factor` and object-level sleep is `time spent on the object x factor`, both capped at `max_wait`; a foreground-read floor of `FOREGROUND_READ_BACKOFF_PER_REQUEST_MS` (10ms) per concurrent GetObject/streaming read, capped at `FOREGROUND_READ_BACKOFF_MAX_MS` (250ms), is applied on top and can exceed the preset's `max_wait` (`crates/scanner/src/sleeper.rs`). The per-disk foreground-priority gate is independent of `idle_mode` and remains active when sleeps are disabled.
|
||||
Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. With `idle_mode=true`, directory-level sleep is `1ms x factor` and object-level sleep is admitted work time multiplied by the factor, capped at `max_wait`; object pacing batches small waits and settles remaining debt at bucket completion. Local walkers bound with `DynamicSleeper::for_disk` use disk-priority admission and their shared I/O budget without an additional global foreground-request sleep. A caller without that binding retains a foreground-workload floor of 10ms per active request, capped at 250ms, which can exceed `max_wait` (`crates/scanner/src/sleeper.rs`). The per-disk foreground-priority gate and disk budget remain active when preset sleeps are disabled. Removing redundant global backoff does not guarantee a fixed snapshot age: scans still compete for admitted resources and may checkpoint at their cycle budget.
|
||||
|
||||
### Startup and cycle budgets
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ because object identity is unproven; that is distinct from execution progress.
|
||||
MRF likewise retains durable legacy repair obligations after physical recovery
|
||||
when no independent verification receipt can discharge them. An idle attempt
|
||||
queue does not imply that these persistent obligations have been cleared.
|
||||
|
||||
After a qualified all-healthy Deep check, the attempt is `Completed` with an `Unknown` object outcome; it is counted in `unverified_tasks`, not `successful_tasks` or `failed_tasks`. This cumulative attempt count is distinct from the current `held_unverified_legacy` responsibility count. The scheduler reports `verification_unavailable`, and the durable obligation remains subject to the existing explicit recheck or audited operator disposition. Execution completion never supplies a missing integrity receipt.
|
||||
|
||||
Partial-write MRF replay uses Deep verification so a protected object's repair
|
||||
can discharge its obligation after verifying the payload. This adds full-object
|
||||
read work to those background attempts, including healthy replay targets.
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1793466162,
|
||||
"narHash": "sha256-pt+peAEoOK3lLLdtbnQ1AGIBTZbdB6jrkgmurfcQwig=",
|
||||
"lastModified": 1793977579,
|
||||
"narHash": "sha256-WrSODdZFVIs/Hh2NzC0bY+ROByi0bATg8WfePX8dPj8=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a0788123ada57ec92a79ab2581cda6d3c6cca75f",
|
||||
"rev": "4fe25e12eb9f4c744966dd3de5a05246cf098601",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -29,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1793434469,
|
||||
"narHash": "sha256-+sgVNI2c67di+XIeMaz5KSei4T65kX68s5Nl0E5Gzbc=",
|
||||
"lastModified": 1794040562,
|
||||
"narHash": "sha256-7eEiezSku3lbZPUOXW8we1a7+xd5axkCQdyI3rzOKjo=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "6ecf534a7c16e7c969f26838fdd8c93597ae93f3",
|
||||
"rev": "d3fe74269f2a6652437e21e78eb7e83be81f6982",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -211,9 +211,7 @@ pub(crate) struct ClusterPoolMetaWriteGateView {
|
||||
|
||||
impl From<ClusterPoolMetaWriteGateSnapshot> for ClusterPoolMetaWriteGateView {
|
||||
fn from(snapshot: ClusterPoolMetaWriteGateSnapshot) -> Self {
|
||||
let state = if snapshot.check_timed_out {
|
||||
"check_timeout"
|
||||
} else if snapshot.writes_ready {
|
||||
let state = if snapshot.writes_ready {
|
||||
"writable"
|
||||
} else if snapshot.write_blocked || snapshot.transaction_aborted {
|
||||
"blocked"
|
||||
@@ -706,7 +704,6 @@ fn summarize_storage_readiness(snapshot: &ClusterReadOnlySnapshot) -> Capability
|
||||
.filter_map(|reason| match reason {
|
||||
ReadinessDegradedReason::StorageQuorumUnavailable
|
||||
| ReadinessDegradedReason::PoolMetaWriteBlocked
|
||||
| ReadinessDegradedReason::PoolMetadataCheckTimeout
|
||||
| ReadinessDegradedReason::StorageAndIamUnavailable
|
||||
| ReadinessDegradedReason::StorageAndLockUnavailable
|
||||
| ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()),
|
||||
@@ -1120,14 +1117,6 @@ mod tests {
|
||||
|
||||
for (snapshot, state) in [
|
||||
(ClusterPoolMetaWriteGateSnapshot::default(), "writable"),
|
||||
(
|
||||
ClusterPoolMetaWriteGateSnapshot {
|
||||
writes_ready: false,
|
||||
check_timed_out: true,
|
||||
..Default::default()
|
||||
},
|
||||
"check_timeout",
|
||||
),
|
||||
(
|
||||
ClusterPoolMetaWriteGateSnapshot {
|
||||
writes_ready: false,
|
||||
|
||||
@@ -88,67 +88,6 @@ impl Operation for GetTableBucketHandler {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GetTableCatalogMigrationHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for GetTableCatalogMigrationHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.plan_durable_strong_backing_migration(&warehouse)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("migration", &warehouse, "", "", started, &result);
|
||||
let response = result?;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MaterializeTableCatalogMigrationHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for MaterializeTableCatalogMigrationHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.materialize_durable_strong_backing_migration(&warehouse)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result);
|
||||
let response = result?;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CancelTableCatalogMigrationHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for CancelTableCatalogMigrationHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.cancel_durable_strong_backing_migration(&warehouse)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result);
|
||||
let response = result?;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CreateTableCatalogBackupHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -162,7 +101,7 @@ impl Operation for CreateTableCatalogBackupHandler {
|
||||
let store = table_catalog_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.create_durable_catalog_backup(&warehouse, request.expected_snapshot_etag.as_deref())
|
||||
.create_catalog_backup(&warehouse, request.expected_snapshot_etag.as_deref())
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("catalog-backup", &warehouse, "", "", started, &result);
|
||||
@@ -171,6 +110,98 @@ impl Operation for CreateTableCatalogBackupHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the saved artifact bytes without applying or re-encoding its catalog.
|
||||
pub struct ExportTableCatalogBackupHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ExportTableCatalogBackupHandler {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let request = read_json_body::<CatalogBackupExportRequest>(std::mem::take(&mut req.input)).await?;
|
||||
let store = table_catalog_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.export_catalog_backup(&warehouse, &request.backup_id)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("catalog-backup-export", &warehouse, "", "", started, &result);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE));
|
||||
headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store, private"));
|
||||
headers.insert(http::header::PRAGMA, HeaderValue::from_static("no-cache"));
|
||||
headers.insert(http::header::EXPIRES, HeaderValue::from_static("0"));
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(result?)), headers))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores a validated candidate artifact; catalog publication is a separate restore.
|
||||
pub struct ImportTableCatalogBackupHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ImportTableCatalogBackupHandler {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let data = read_limited_body(
|
||||
std::mem::take(&mut req.input),
|
||||
CATALOG_BACKUP_IMPORT_BODY_MAX_SIZE,
|
||||
TABLE_CATALOG_REQUEST_BODY_TIMEOUT,
|
||||
Some("catalog backup import"),
|
||||
)
|
||||
.await?;
|
||||
let store = table_catalog_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.import_catalog_backup(&warehouse, data.to_vec())
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("catalog-backup-import", &warehouse, "", "", started, &result);
|
||||
build_sensitive_json_response(StatusCode::OK, &result?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Prepares one verified small object while leaving catalog publication to restore.
|
||||
pub struct PrepareTableCatalogBackupObjectHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for PrepareTableCatalogBackupObjectHandler {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let request = read_bounded_json_body::<CatalogBackupObjectPrepareRequest>(
|
||||
&req.headers,
|
||||
std::mem::take(&mut req.input),
|
||||
CATALOG_BACKUP_PREPARE_BODY_MAX_SIZE,
|
||||
TABLE_CATALOG_REQUEST_BODY_TIMEOUT,
|
||||
"catalog backup object preparation",
|
||||
)
|
||||
.await?;
|
||||
if request.data_base64.len() > CATALOG_BACKUP_PREPARE_BASE64_MAX_SIZE {
|
||||
return Err(S3Error::from(ApiError::invalid_request(
|
||||
"data-base64 exceeds the encoded preparation limit",
|
||||
)));
|
||||
}
|
||||
let data = base64_simd::STANDARD
|
||||
.decode_to_vec(request.data_base64.as_bytes())
|
||||
.map_err(|_| S3Error::from(ApiError::invalid_request("data-base64 must be standard padded Base64")))?;
|
||||
let store = table_catalog_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.prepare_catalog_backup_object(&warehouse, &request.backup_id, &request.object, data)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
record_table_catalog_admin_operation_result("catalog-backup-prepare", &warehouse, "", "", started, &result);
|
||||
build_sensitive_json_response(StatusCode::OK, &result?)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RestoreTableCatalogBackupHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -184,11 +215,12 @@ impl Operation for RestoreTableCatalogBackupHandler {
|
||||
let store = table_catalog_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
let result = store
|
||||
.restore_durable_catalog_backup(
|
||||
.restore_catalog_backup_with_id(
|
||||
&warehouse,
|
||||
&request.backup_id,
|
||||
request.expected_snapshot_etag.as_deref(),
|
||||
request.allow_replace,
|
||||
request.restore_id.as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
@@ -204,7 +236,8 @@ pub struct BackfillTableWarehouseIndexHandler {}
|
||||
impl Operation for BackfillTableWarehouseIndexHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let started = Instant::now();
|
||||
@@ -315,7 +348,8 @@ pub struct AdvanceTableDirectoryHandler {}
|
||||
impl Operation for AdvanceTableDirectoryHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let response = store.advance_table_directory(&warehouse).await.map_err(catalog_store_error)?;
|
||||
@@ -329,7 +363,8 @@ pub struct DisableTableDirectoryHandler {}
|
||||
impl Operation for DisableTableDirectoryHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
|
||||
let response = store.disable_table_directory(&warehouse).await.map_err(catalog_store_error)?;
|
||||
|
||||
@@ -46,30 +46,8 @@ async fn configured_table_metadata_maintenance_response<B>(
|
||||
where
|
||||
B: crate::table_catalog::TableCatalogObjectBackend + Clone,
|
||||
{
|
||||
match store {
|
||||
crate::table_catalog::ConfiguredTableCatalogStore::ObjectBacked(store) => {
|
||||
table_metadata_maintenance_response(store, metadata_backend, bucket, namespace, table, request).await
|
||||
}
|
||||
crate::table_catalog::ConfiguredTableCatalogStore::DurableStrong(store) => {
|
||||
if request.delete || request.commit_snapshot_expiration || request.commit_compaction {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"metadata maintenance deletion and snapshot expiration or compaction commits are not supported with durable-strong table catalog backing"
|
||||
));
|
||||
}
|
||||
store
|
||||
.run_table_metadata_maintenance_planning(
|
||||
bucket,
|
||||
&namespace.public_name(),
|
||||
table,
|
||||
request.retain_recent_metadata_files,
|
||||
request.snapshot_expiration,
|
||||
request.compaction,
|
||||
)
|
||||
.await
|
||||
.map_err(catalog_store_error)
|
||||
}
|
||||
}
|
||||
let crate::table_catalog::ConfiguredTableCatalogStore::ObjectBacked(store) = store;
|
||||
table_metadata_maintenance_response(store, metadata_backend, bucket, namespace, table, request).await
|
||||
}
|
||||
|
||||
pub struct GetTableMaintenanceConfigHandler {}
|
||||
|
||||
@@ -79,6 +79,10 @@ const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60;
|
||||
const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60;
|
||||
const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60;
|
||||
const TABLE_CATALOG_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(30);
|
||||
const CATALOG_BACKUP_IMPORT_BODY_MAX_SIZE: usize = 64 * 1024 * 1024;
|
||||
const CATALOG_BACKUP_PREPARE_BODY_MAX_SIZE: usize = 24 * 1024 * 1024;
|
||||
const CATALOG_BACKUP_PREPARE_OBJECT_MAX_SIZE: usize = 16 * 1024 * 1024;
|
||||
const CATALOG_BACKUP_PREPARE_BASE64_MAX_SIZE: usize = CATALOG_BACKUP_PREPARE_OBJECT_MAX_SIZE.div_ceil(3) * 4;
|
||||
const TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT: usize = 1_024;
|
||||
const TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT: usize = 1_024;
|
||||
const NAMESPACE_REQUEST_BODY_MAX_SIZE: usize = MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
@@ -177,18 +181,15 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
|
||||
"POST /v1/{prefix}/namespaces/{namespace}/views/{view}",
|
||||
"DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}",
|
||||
];
|
||||
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] =
|
||||
&["POST /v1/{prefix}/catalog/backup", "POST /v1/{prefix}/catalog/restore"];
|
||||
|
||||
static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
|
||||
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
|
||||
static DISABLE_TABLE_BUCKET_HANDLER: DisableTableBucketHandler = DisableTableBucketHandler {};
|
||||
static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {};
|
||||
static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {};
|
||||
static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler =
|
||||
MaterializeTableCatalogMigrationHandler {};
|
||||
static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {};
|
||||
static CREATE_TABLE_CATALOG_BACKUP_HANDLER: CreateTableCatalogBackupHandler = CreateTableCatalogBackupHandler {};
|
||||
static EXPORT_TABLE_CATALOG_BACKUP_HANDLER: ExportTableCatalogBackupHandler = ExportTableCatalogBackupHandler {};
|
||||
static IMPORT_TABLE_CATALOG_BACKUP_HANDLER: ImportTableCatalogBackupHandler = ImportTableCatalogBackupHandler {};
|
||||
static PREPARE_TABLE_CATALOG_BACKUP_OBJECT_HANDLER: PrepareTableCatalogBackupObjectHandler =
|
||||
PrepareTableCatalogBackupObjectHandler {};
|
||||
static RESTORE_TABLE_CATALOG_BACKUP_HANDLER: RestoreTableCatalogBackupHandler = RestoreTableCatalogBackupHandler {};
|
||||
static BACKFILL_TABLE_WAREHOUSE_INDEX_HANDLER: BackfillTableWarehouseIndexHandler = BackfillTableWarehouseIndexHandler {};
|
||||
static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {};
|
||||
@@ -513,11 +514,30 @@ struct CatalogBackupRequest {
|
||||
expected_snapshot_etag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct CatalogBackupExportRequest {
|
||||
#[serde(rename = "backup-id")]
|
||||
backup_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct CatalogBackupObjectPrepareRequest {
|
||||
#[serde(rename = "backup-id")]
|
||||
backup_id: String,
|
||||
object: String,
|
||||
#[serde(rename = "data-base64")]
|
||||
data_base64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct CatalogRestoreRequest {
|
||||
#[serde(rename = "backup-id")]
|
||||
backup_id: String,
|
||||
#[serde(default, rename = "restore-id")]
|
||||
restore_id: Option<String>,
|
||||
#[serde(default, rename = "expected-snapshot-etag")]
|
||||
expected_snapshot_etag: Option<String>,
|
||||
#[serde(default, rename = "allow-replace")]
|
||||
@@ -1037,10 +1057,7 @@ fn catalog_config_response(warehouse: Option<&str>) -> S3Result<CatalogConfigRes
|
||||
if let Some(warehouse) = warehouse {
|
||||
defaults.insert(PREFIX_PROPERTY.to_string(), warehouse.to_string());
|
||||
}
|
||||
let mut endpoints = TABLE_CATALOG_ENDPOINTS.to_vec();
|
||||
if backing_mode == crate::table_catalog::TableCatalogBackingMode::DurableStrong {
|
||||
endpoints.extend_from_slice(TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS);
|
||||
}
|
||||
let endpoints = TABLE_CATALOG_ENDPOINTS.to_vec();
|
||||
Ok(CatalogConfigResponse {
|
||||
defaults,
|
||||
overrides,
|
||||
@@ -2093,10 +2110,6 @@ where
|
||||
&& publication.guards.iter().all(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
fn acquires_catalog_migration_read_permit(&self) -> bool {
|
||||
self.backend.acquires_catalog_migration_read_permit()
|
||||
}
|
||||
|
||||
fn catalog_migration_read_permit_status(&self) -> Option<bool> {
|
||||
let publication = self.publication.lock();
|
||||
publication
|
||||
@@ -2579,9 +2592,8 @@ fn table_catalog_backend_from_extensions(
|
||||
) -> S3Result<crate::table_catalog::EcStoreTableCatalogObjectBackend<ECStore>> {
|
||||
let context = runtime_sources::app_context_from_extensions(extensions)
|
||||
.ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?;
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_runtimes(
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_registration_cache(
|
||||
context.object_store(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
context.table_catalog_registration_cache(),
|
||||
))
|
||||
}
|
||||
@@ -2602,16 +2614,10 @@ fn table_catalog_store_from_extensions(
|
||||
}
|
||||
|
||||
fn table_catalog_object_store_from_extensions(extensions: &http::Extensions) -> S3Result<EcStoreObjectTableCatalogStore> {
|
||||
match crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)? {
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked => Ok(crate::table_catalog::ObjectTableCatalogStore::new(
|
||||
table_catalog_backend_from_extensions(extensions)?,
|
||||
)),
|
||||
crate::table_catalog::TableCatalogBackingMode::DurableStrong => Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"operation is not supported with {} table catalog backing",
|
||||
crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG
|
||||
)),
|
||||
}
|
||||
crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)?;
|
||||
Ok(crate::table_catalog::ObjectTableCatalogStore::new(table_catalog_backend_from_extensions(
|
||||
extensions,
|
||||
)?))
|
||||
}
|
||||
|
||||
async fn table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result<bool> {
|
||||
|
||||
@@ -54,26 +54,26 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
|
||||
format!("{prefix}/{{warehouse}}/catalog/directory").as_str(),
|
||||
AdminOperation(&DisableTableDirectoryHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
|
||||
AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/backup").as_str(),
|
||||
AdminOperation(&CREATE_TABLE_CATALOG_BACKUP_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/backup/export").as_str(),
|
||||
AdminOperation(&EXPORT_TABLE_CATALOG_BACKUP_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/backup/import").as_str(),
|
||||
AdminOperation(&IMPORT_TABLE_CATALOG_BACKUP_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/backup/prepare").as_str(),
|
||||
AdminOperation(&PREPARE_TABLE_CATALOG_BACKUP_OBJECT_HANDLER),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{prefix}/{{warehouse}}/catalog/restore").as_str(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,306 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::storage::storage_api::contract::bucket::BucketOperations as _;
|
||||
use crate::table_catalog::TableCatalogObject;
|
||||
use rustfs_iam::store::Store as _;
|
||||
|
||||
const SCOPED_WAREHOUSE: &str = "migration-scope-owned";
|
||||
const FOREIGN_WAREHOUSE: &str = "migration-scope-foreign";
|
||||
const UNMARKED_WAREHOUSE: &str = "migration-scope-unmarked";
|
||||
const SCOPED_USER: &str = "catalog-migration-scoped-user";
|
||||
const SCOPED_SECRET: &str = "catalog-migration-scoped-user-secret";
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum CatalogMigrationScopeOperation {
|
||||
Backfill,
|
||||
Backup,
|
||||
Restore,
|
||||
}
|
||||
|
||||
impl CatalogMigrationScopeOperation {
|
||||
fn suffix(self) -> &'static str {
|
||||
match self {
|
||||
Self::Backfill => "/catalog/warehouse-index/backfill",
|
||||
Self::Backup => "/catalog/backup",
|
||||
Self::Restore => "/catalog/restore",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CatalogMigrationScopeFixture {
|
||||
_temp_dir: tempfile::TempDir,
|
||||
context: Arc<AppContext>,
|
||||
}
|
||||
|
||||
impl CatalogMigrationScopeFixture {
|
||||
async fn new() -> Self {
|
||||
let (temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
for bucket in [SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
|
||||
object_store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create isolated scope fixture bucket");
|
||||
}
|
||||
rustfs_iam::store::object::ObjectStore::new(object_store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.expect("seed isolated IAM format");
|
||||
let iam = rustfs_iam::build_iam_sys(object_store.clone())
|
||||
.await
|
||||
.expect("build isolated request IAM");
|
||||
let context = Arc::new(AppContext::new(
|
||||
object_store.clone(),
|
||||
Arc::new(RequestIam { handle: iam.clone() }),
|
||||
Arc::new(RequestKms),
|
||||
));
|
||||
// Preserve the existing process-global winner. When this test publishes
|
||||
// first, use the same root as the existing load-table fixture.
|
||||
let mut root = rustfs_credentials::get_global_action_cred().unwrap_or_else(|| rustfs_credentials::Credentials {
|
||||
access_key: "load-table-root-access-key".to_string(),
|
||||
secret_key: "load-table-root-secret-key".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
root.status = "on".to_string();
|
||||
assert!(context.publish_action_credentials(root));
|
||||
iam.create_user(
|
||||
SCOPED_USER,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: SCOPED_SECRET.to_string(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a regular non-owner migration user");
|
||||
let policy = serde_json::to_vec(&serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": "admin:MigrateTableCatalog",
|
||||
"Resource": format!("arn:aws:s3:::{SCOPED_WAREHOUSE}")
|
||||
}]
|
||||
}))
|
||||
.expect("encode warehouse-limited migration policy");
|
||||
iam.set_policy(
|
||||
"catalog-migration-resource-scope",
|
||||
Policy::parse_config(&policy).expect("parse warehouse-limited migration policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store migration policy");
|
||||
iam.policy_db_set(SCOPED_USER, UserType::Reg, false, "catalog-migration-resource-scope")
|
||||
.await
|
||||
.expect("attach migration policy to the regular user");
|
||||
|
||||
let setup = table_catalog_handler_request(context.clone(), SCOPED_USER, SCOPED_SECRET);
|
||||
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve the isolated catalog backend");
|
||||
let catalog = table_catalog_store_from_backend(backend).expect("resolve object catalog");
|
||||
for bucket in [SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE] {
|
||||
enable_table_bucket_marker(object_store.as_ref(), bucket)
|
||||
.await
|
||||
.expect("enable fixture table bucket marker");
|
||||
ensure_table_bucket_entry(&catalog, bucket, true)
|
||||
.await
|
||||
.expect("seed fixture catalog bucket");
|
||||
}
|
||||
Self {
|
||||
_temp_dir: temp_dir,
|
||||
context,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(&self, prefix: &str, warehouse: &str, operation: CatalogMigrationScopeOperation, body: &[u8]) -> S3Request<Body> {
|
||||
let mut request = table_catalog_handler_request(self.context.clone(), SCOPED_USER, SCOPED_SECRET);
|
||||
request.method = Method::POST;
|
||||
request.uri = format!("{prefix}/{warehouse}{}", operation.suffix())
|
||||
.parse()
|
||||
.expect("scope handler URI");
|
||||
request
|
||||
.headers
|
||||
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
request.input = Body::from(body.to_vec());
|
||||
request
|
||||
}
|
||||
|
||||
async fn inventory(&self) -> BTreeMap<(String, String), TableCatalogObject> {
|
||||
let setup = table_catalog_handler_request(self.context.clone(), SCOPED_USER, SCOPED_SECRET);
|
||||
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve inventory backend");
|
||||
let mut result = BTreeMap::new();
|
||||
for bucket in [RUSTFS_META_BUCKET, SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
|
||||
for object in backend.list_objects(bucket, "").await.expect("list isolated fixture objects") {
|
||||
let stored = backend
|
||||
.read_object(bucket, &object)
|
||||
.await
|
||||
.expect("read isolated fixture object")
|
||||
.expect("listed fixture object exists");
|
||||
result.insert((bucket.to_string(), object), stored);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_catalog_migration_scope_handler(
|
||||
request: S3Request<Body>,
|
||||
prefix: &str,
|
||||
operation: CatalogMigrationScopeOperation,
|
||||
) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router
|
||||
.insert(format!("{prefix}/{{warehouse}}{}", operation.suffix()), ())
|
||||
.expect("insert the real catalog route shape");
|
||||
let path = request.uri.path().to_string();
|
||||
let matched = router.at(&path).expect("match the warehouse parameter");
|
||||
match operation {
|
||||
CatalogMigrationScopeOperation::Backfill => BackfillTableWarehouseIndexHandler {}.call(request, matched.params).await,
|
||||
CatalogMigrationScopeOperation::Backup => CreateTableCatalogBackupHandler {}.call(request, matched.params).await,
|
||||
CatalogMigrationScopeOperation::Restore => RestoreTableCatalogBackupHandler {}.call(request, matched.params).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn catalog_migration_scope_response_json(response: S3Response<(StatusCode, Body)>) -> serde_json::Value {
|
||||
assert_eq!(response.output.0, StatusCode::OK);
|
||||
let bytes = http_body_util::BodyExt::collect(response.output.1)
|
||||
.await
|
||||
.expect("collect successful catalog response")
|
||||
.to_bytes();
|
||||
serde_json::from_slice(&bytes).expect("parse successful catalog response JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn catalog_backfill_handler_enforces_warehouse_scope_before_marker_or_store_access() {
|
||||
let fixture = CatalogMigrationScopeFixture::new().await;
|
||||
for prefix in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX] {
|
||||
let operation = CatalogMigrationScopeOperation::Backfill;
|
||||
let request = fixture.request(prefix, SCOPED_WAREHOUSE, operation, b"");
|
||||
let principal = authorize_table_catalog_resource_request(
|
||||
&request,
|
||||
&TableCatalogResource::warehouse(SCOPED_WAREHOUSE),
|
||||
AdminAction::MigrateTableCatalogAction,
|
||||
)
|
||||
.await
|
||||
.expect("the scoped policy allows its own warehouse");
|
||||
assert!(!principal.owner, "scope checks must use the regular user rather than owner bypass");
|
||||
let response = call_catalog_migration_scope_handler(request, prefix, operation)
|
||||
.await
|
||||
.expect("authorized warehouse-index backfill succeeds");
|
||||
assert_eq!(response.output.0, StatusCode::NO_CONTENT);
|
||||
|
||||
for warehouse in [FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
|
||||
let before = fixture.inventory().await;
|
||||
let request = fixture.request(prefix, warehouse, operation, b"{body must not be consumed");
|
||||
let error = call_catalog_migration_scope_handler(request, prefix, operation)
|
||||
.await
|
||||
.expect_err("a different warehouse must be denied before marker or store access");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{prefix} {warehouse}");
|
||||
assert_eq!(error.status_code(), Some(StatusCode::FORBIDDEN), "{prefix} {warehouse}");
|
||||
assert_eq!(fixture.inventory().await, before, "denied backfill changed persistent state");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn catalog_backup_restore_enforce_migration_scope_and_replay_own_receipts() {
|
||||
let fixture = CatalogMigrationScopeFixture::new().await;
|
||||
for (index, prefix) in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX].into_iter().enumerate() {
|
||||
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Backup, b"{}");
|
||||
let principal = table_catalog_request_principal(&request)
|
||||
.await
|
||||
.expect("authenticate the scoped backup caller");
|
||||
assert!(!principal.owner, "backup and restore must not use owner bypass");
|
||||
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Backup)
|
||||
.await
|
||||
.expect("the scoped user can create its own object catalog backup");
|
||||
let backup = catalog_migration_scope_response_json(response).await;
|
||||
assert_eq!(backup["table_bucket"], SCOPED_WAREHOUSE);
|
||||
assert!(matches!(backup["status"].as_str(), Some("CREATED" | "ALREADY_PRESENT")));
|
||||
assert!(backup["object_count"].as_u64().is_some_and(|count| count > 0));
|
||||
let backup_id = backup["backup_id"].as_str().expect("successful backup id").to_string();
|
||||
assert_eq!(backup_id.len(), 64);
|
||||
assert!(
|
||||
backup_id
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
);
|
||||
let watermark = backup["source_snapshot_etag"]
|
||||
.as_str()
|
||||
.expect("successful backup source watermark");
|
||||
assert!(!watermark.is_empty());
|
||||
let restore_id = format!("scoped-handler-restore-{index}");
|
||||
let restore_body = serde_json::to_vec(&serde_json::json!({
|
||||
"backup-id": backup_id,
|
||||
"restore-id": restore_id,
|
||||
"expected-snapshot-etag": watermark,
|
||||
"allow-replace": false
|
||||
}))
|
||||
.expect("bind the restore request to the successful backup and explicit restore id");
|
||||
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Restore, &restore_body);
|
||||
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Restore)
|
||||
.await
|
||||
.expect("the scoped user can restore the unchanged catalog from its own backup");
|
||||
let restored = catalog_migration_scope_response_json(response).await;
|
||||
assert_eq!(restored["table_bucket"], SCOPED_WAREHOUSE);
|
||||
assert_eq!(restored["backup_id"], backup_id);
|
||||
assert_eq!(restored["status"], "RESTORED");
|
||||
assert_eq!(restored["object_count"], backup["object_count"]);
|
||||
assert_eq!(restored["maintenance_object_count"], backup["maintenance_object_count"]);
|
||||
|
||||
let setup = table_catalog_handler_request(fixture.context.clone(), SCOPED_USER, SCOPED_SECRET);
|
||||
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve restore control backend");
|
||||
let state_path = crate::table_catalog::TableCatalogObjectPaths::default().catalog_restore_state_path(SCOPED_WAREHOUSE);
|
||||
let barrier = backend
|
||||
.read_object(RUSTFS_META_BUCKET, &state_path)
|
||||
.await
|
||||
.expect("read identified restore barrier")
|
||||
.expect("identified restore barrier exists");
|
||||
let barrier: serde_json::Value = serde_json::from_slice(&barrier.data).expect("identified restore barrier JSON");
|
||||
assert_eq!(barrier["restore_id"], restore_id, "handler did not forward the explicit restore id");
|
||||
assert_eq!(barrier["backup_id"], backup_id);
|
||||
assert_eq!(barrier["state"], "COMPLETED");
|
||||
|
||||
let before_replay = fixture.inventory().await;
|
||||
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Restore, &restore_body);
|
||||
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Restore)
|
||||
.await
|
||||
.expect("the same restore id replays the completed receipt");
|
||||
let mut replay = catalog_migration_scope_response_json(response).await;
|
||||
assert_eq!(replay["status"], "ALREADY_RESTORED");
|
||||
replay["status"] = restored["status"].clone();
|
||||
assert_eq!(replay, restored, "replay changed the original receipt");
|
||||
assert_eq!(fixture.inventory().await, before_replay, "completed replay changed persistent state");
|
||||
|
||||
for warehouse in [FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
|
||||
for operation in [
|
||||
CatalogMigrationScopeOperation::Backup,
|
||||
CatalogMigrationScopeOperation::Restore,
|
||||
] {
|
||||
let before = fixture.inventory().await;
|
||||
let request = fixture.request(prefix, warehouse, operation, b"{body must not be consumed");
|
||||
let error = call_catalog_migration_scope_handler(request, prefix, operation)
|
||||
.await
|
||||
.expect_err("foreign warehouse must fail authorization before malformed JSON or marker checks");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{prefix} {warehouse} {operation:?}");
|
||||
assert_eq!(error.status_code(), Some(StatusCode::FORBIDDEN), "{prefix} {warehouse} {operation:?}");
|
||||
assert_eq!(fixture.inventory().await, before, "foreign backup/restore changed persistent state");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1150,24 +1150,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/iceberg/v1/{warehouse}/catalog/migration",
|
||||
GET_TABLE_CATALOG,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/backup",
|
||||
@@ -1180,6 +1162,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/backup/export",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/backup/import",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/backup/prepare",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
|
||||
@@ -1476,24 +1476,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/_iceberg/v1/{warehouse}/catalog/migration",
|
||||
GET_TABLE_CATALOG,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/_iceberg/v1/{warehouse}/catalog/migration",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/backup",
|
||||
@@ -1506,6 +1488,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/backup/export",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/backup/import",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/backup/prepare",
|
||||
MIGRATE_TABLE_CATALOG,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/_iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
|
||||
@@ -2346,16 +2346,16 @@ mod tests {
|
||||
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/directory", GET_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/export", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/export", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/import", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/import", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/prepare", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/prepare", MIGRATE_TABLE_CATALOG);
|
||||
assert_action(
|
||||
HttpMethod::Post,
|
||||
"/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
|
||||
|
||||
@@ -511,14 +511,14 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
table_route_sample(Method::DELETE, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
table_route_sample(Method::GET, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
table_route_sample(Method::DELETE, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/export", "/analytics/catalog/backup/export"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/import", "/analytics/catalog/backup/import"),
|
||||
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/prepare", "/analytics/catalog/backup/prepare"),
|
||||
table_route_sample(
|
||||
Method::POST,
|
||||
"/{warehouse}/catalog/warehouse-index/backfill",
|
||||
@@ -719,14 +719,14 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
compat_table_route_sample(Method::DELETE, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
|
||||
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
|
||||
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/export", "/analytics/catalog/backup/export"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/import", "/analytics/catalog/backup/import"),
|
||||
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/prepare", "/analytics/catalog/backup/prepare"),
|
||||
compat_table_route_sample(
|
||||
Method::POST,
|
||||
"/{warehouse}/catalog/warehouse-index/backfill",
|
||||
|
||||
@@ -75,7 +75,6 @@ pub struct AppContext {
|
||||
buffer_config: Arc<dyn BufferConfigInterface>,
|
||||
object_data_cache: Arc<ObjectDataCacheAdapter>,
|
||||
object_traffic_health: Arc<ObjectTrafficHealth>,
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime,
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
@@ -127,7 +126,6 @@ impl AppContext {
|
||||
buffer_config: default_buffer_config_interface(),
|
||||
object_data_cache,
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
@@ -148,14 +146,9 @@ impl AppContext {
|
||||
Arc::clone(&self.object_traffic_health)
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_strong_runtime(&self) -> crate::table_catalog::StrongTableCatalogRuntime {
|
||||
self.table_catalog_strong_runtime.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_registration_cache(&self) -> crate::table_catalog::TableCatalogRegistrationCache {
|
||||
self.table_catalog_registration_cache.clone()
|
||||
}
|
||||
|
||||
pub fn iam(&self) -> Arc<dyn IamInterface> {
|
||||
self.iam.clone()
|
||||
}
|
||||
@@ -365,7 +358,6 @@ impl AppContext {
|
||||
buffer_config: interfaces.buffer_config,
|
||||
object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -902,8 +902,7 @@ mod tests {
|
||||
status: "on".to_owned(),
|
||||
..Default::default()
|
||||
}));
|
||||
let backend =
|
||||
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(Arc::clone(&store), context.table_catalog_strong_runtime());
|
||||
let backend = EcStoreTableCatalogObjectBackend::new(Arc::clone(&store));
|
||||
let catalog =
|
||||
ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::from_env().expect("catalog mode"));
|
||||
catalog
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user