mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-09 20:08:12 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 509869cfc5 | |||
| 395876ccd0 | |||
| 6083fbbeea | |||
| 42d3267d88 | |||
| 1d35e381b7 | |||
| 0b109b4885 | |||
| 1dea00ffa7 |
@@ -1,2 +1,2 @@
|
|||||||
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
|
sha256-darwin=9901a4950099b94d3d2618524d540f1ed4bf6023044903886f98b6c4bf5f30fb
|
||||||
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
|
sha256-linux=09fd5f48f5556c9fc576357222dccd7ebb0b90b790798f1834db2ec5d67b4952
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ script-tests: ## Run shell script tests
|
|||||||
./scripts/check_embedded_secrets.sh --self-test
|
./scripts/check_embedded_secrets.sh --self-test
|
||||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||||
|
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||||
|
|||||||
@@ -28,11 +28,12 @@ runs:
|
|||||||
|
|
||||||
- name: Install actionlint
|
- name: Install actionlint
|
||||||
shell: bash
|
shell: bash
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
|
||||||
run: |
|
run: |
|
||||||
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
|
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
|
||||||
curl --fail --location --silent --show-error \
|
python3 "$ASSET_DOWNLOADER" --repository rhysd/actionlint --tag v1.7.12 --asset actionlint_1.7.12_linux_amd64.tar.gz --output "$actionlint_dir/actionlint.tar.gz" --sha256 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
|
||||||
--output "$actionlint_dir/actionlint.tar.gz" \
|
|
||||||
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
|
|
||||||
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
|
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
|
||||||
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
|
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
|
||||||
rm "$actionlint_dir/actionlint.tar.gz"
|
rm "$actionlint_dir/actionlint.tar.gz"
|
||||||
|
|||||||
@@ -66,10 +66,7 @@ inputs:
|
|||||||
runs:
|
runs:
|
||||||
using: "composite"
|
using: "composite"
|
||||||
steps:
|
steps:
|
||||||
# protobuf-compiler is deliberately absent: the setup-protoc step below
|
# Use the pinned code generators rather than Ubuntu's older packages.
|
||||||
# installs 35.1 into the tool cache and prepends it to PATH, so the apt
|
|
||||||
# build (older, and never version-matched) was shadowed on every run and
|
|
||||||
# simply never used.
|
|
||||||
- name: Install system dependencies (Ubuntu)
|
- name: Install system dependencies (Ubuntu)
|
||||||
if: runner.os == 'Linux'
|
if: runner.os == 'Linux'
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -79,7 +76,9 @@ runs:
|
|||||||
build-essential \
|
build-essential \
|
||||||
pkg-config \
|
pkg-config \
|
||||||
libssl-dev \
|
libssl-dev \
|
||||||
ripgrep
|
ripgrep \
|
||||||
|
python3 \
|
||||||
|
gh
|
||||||
|
|
||||||
# musl-gcc is needed by the native musl release leg, and zip/unzip by the
|
# musl-gcc is needed by the native musl release leg, and zip/unzip by the
|
||||||
# release packaging steps. No CI test lane touches any of them.
|
# release packaging steps. No CI test lane touches any of them.
|
||||||
@@ -88,13 +87,32 @@ runs:
|
|||||||
shell: bash
|
shell: bash
|
||||||
run: sudo apt-get install -y musl-tools zip unzip
|
run: sudo apt-get install -y musl-tools zip unzip
|
||||||
|
|
||||||
|
- name: Install pinned code generators through GitHub API (Linux x64)
|
||||||
|
if: runner.os == 'Linux' && runner.arch == 'X64'
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
ASSET_DOWNLOADER: ${{ github.action_path }}/download_release_asset.py
|
||||||
|
run: |
|
||||||
|
python3 "$ASSET_DOWNLOADER" --repository protocolbuffers/protobuf --tag v35.1 --asset protoc-35.1-linux-x86_64.zip --output "$RUNNER_TEMP/protoc.zip"
|
||||||
|
python3 "$ASSET_DOWNLOADER" --repository google/flatbuffers --tag v25.12.19 --asset 'Linux.flatc.binary.g++-13.zip' --output "$RUNNER_TEMP/flatc.zip"
|
||||||
|
python3 -m zipfile -e "$RUNNER_TEMP/protoc.zip" "$RUNNER_TEMP/protoc"
|
||||||
|
python3 -m zipfile -e "$RUNNER_TEMP/flatc.zip" "$RUNNER_TEMP/flatc"
|
||||||
|
chmod +x "$RUNNER_TEMP/protoc/bin/protoc" "$RUNNER_TEMP/flatc/flatc"
|
||||||
|
"$RUNNER_TEMP/protoc/bin/protoc" --version
|
||||||
|
"$RUNNER_TEMP/flatc/flatc" --version
|
||||||
|
echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
|
||||||
|
echo "$RUNNER_TEMP/flatc" >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Install protoc
|
- name: Install protoc
|
||||||
|
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||||
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
||||||
with:
|
with:
|
||||||
version: "35.1"
|
version: "35.1"
|
||||||
repo-token: ${{ github.token }}
|
repo-token: ${{ github.token }}
|
||||||
|
|
||||||
- name: Install flatc
|
- name: Install flatc
|
||||||
|
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||||
with:
|
with:
|
||||||
version: "25.12.19"
|
version: "25.12.19"
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Download a named GitHub release asset through its API and verify its digest."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
|
||||||
|
def download(repository, tag, name, output, expected_sha256=None):
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository):
|
||||||
|
raise ValueError("invalid release repository")
|
||||||
|
if not tag or not name or "/" in name or "\\" in name:
|
||||||
|
raise ValueError("invalid release tag or asset name")
|
||||||
|
if expected_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_sha256):
|
||||||
|
raise ValueError("invalid pinned asset digest")
|
||||||
|
result = subprocess.run(
|
||||||
|
["gh", "api", "--hostname", "github.com", f"repos/{repository}/releases/tags/{quote(tag, safe='')}"],
|
||||||
|
check=True, capture_output=True, text=True, timeout=60,
|
||||||
|
)
|
||||||
|
release = json.loads(result.stdout)
|
||||||
|
if not isinstance(release, dict) or release.get("tag_name") != tag:
|
||||||
|
raise ValueError("release metadata does not match the requested tag")
|
||||||
|
assets = release.get("assets")
|
||||||
|
if not isinstance(assets, list) or any(not isinstance(asset, dict) for asset in assets):
|
||||||
|
raise ValueError("invalid release asset inventory")
|
||||||
|
matches = [asset for asset in assets if asset.get("name") == name]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError("release must contain exactly one named asset")
|
||||||
|
asset = matches[0]
|
||||||
|
asset_id, size, digest = asset.get("id"), asset.get("size"), asset.get("digest")
|
||||||
|
if type(asset_id) is not int or asset_id <= 0 or type(size) is not int or not 0 < size <= 512 * 1024 * 1024:
|
||||||
|
raise ValueError("invalid release asset identity or size")
|
||||||
|
if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
||||||
|
raise ValueError("release asset has no valid SHA-256 digest")
|
||||||
|
sha256 = digest.removeprefix("sha256:")
|
||||||
|
if expected_sha256 is not None and expected_sha256 != sha256:
|
||||||
|
raise ValueError("release digest differs from the pinned digest")
|
||||||
|
output = Path(output)
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = None
|
||||||
|
try:
|
||||||
|
with tempfile.NamedTemporaryFile(dir=output.parent, prefix=".release-asset-", delete=False) as stream:
|
||||||
|
temporary = Path(stream.name)
|
||||||
|
# gh authenticates to the API; its HTTP client strips credentials on
|
||||||
|
# cross-host redirects. No browser URL or token is supplied in argv.
|
||||||
|
subprocess.run(
|
||||||
|
["gh", "api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", f"repos/{repository}/releases/assets/{asset_id}"],
|
||||||
|
check=True, stdout=stream, timeout=300,
|
||||||
|
)
|
||||||
|
with temporary.open("rb") as stream:
|
||||||
|
actual = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||||
|
if temporary.stat().st_size != size or actual != sha256:
|
||||||
|
raise ValueError("downloaded release asset failed size or SHA-256 verification")
|
||||||
|
temporary.replace(output)
|
||||||
|
print(f"Verified release asset: {repository} {tag} {name} sha256={actual}")
|
||||||
|
finally:
|
||||||
|
if temporary is not None:
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--repository", required=True)
|
||||||
|
parser.add_argument("--tag", required=True)
|
||||||
|
parser.add_argument("--asset", required=True)
|
||||||
|
parser.add_argument("--output", required=True, type=Path)
|
||||||
|
parser.add_argument("--sha256")
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
download(args.repository, args.tag, args.asset, args.output, args.sha256)
|
||||||
|
except (ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.SubprocessError, OSError) as error:
|
||||||
|
print(f"Release asset download failed: {error}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -142,15 +142,23 @@ jobs:
|
|||||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
- name: Install pinned typos through GitHub API
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
TYPOS_VERSION: 1.50.2
|
||||||
|
run: |
|
||||||
|
python3 .github/actions/setup/download_release_asset.py --repository crate-ci/typos --tag "v${TYPOS_VERSION}" --asset "typos-v${TYPOS_VERSION}-x86_64-unknown-linux-musl.tar.gz" --output "$RUNNER_TEMP/typos.tar.gz"
|
||||||
|
mkdir -p "$RUNNER_TEMP/typos"
|
||||||
|
tar -xzf "$RUNNER_TEMP/typos.tar.gz" -C "$RUNNER_TEMP/typos"
|
||||||
- name: Typos check with custom config file
|
- name: Typos check with custom config file
|
||||||
uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
|
run: '"$RUNNER_TEMP/typos/typos" .'
|
||||||
|
|
||||||
# Fail early with compile-free checks for every pull request.
|
# Fail early with compile-free checks for every pull request.
|
||||||
quick-checks:
|
quick-checks:
|
||||||
name: Quick Checks
|
name: Quick Checks
|
||||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||||
@@ -1137,11 +1145,16 @@ jobs:
|
|||||||
|
|
||||||
- name: Install mc
|
- name: Install mc
|
||||||
env:
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||||
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
||||||
run: |
|
run: |
|
||||||
|
if ! command -v gh >/dev/null 2>&1; then
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y gh
|
||||||
|
fi
|
||||||
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
||||||
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
|
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
|
||||||
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
||||||
chmod +x "$RUNNER_TEMP/mc"
|
chmod +x "$RUNNER_TEMP/mc"
|
||||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||||
|
|||||||
@@ -7,10 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
### Notifications and audit
|
|
||||||
|
|
||||||
- Notify and audit targets that keep the default `queue_dir` now work on native Windows: the defaults `/opt/rustfs/events` and `/opt/rustfs/audit` resolve per node, to `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit` on Windows, instead of failing validation as non-absolute paths. The new `RUSTFS_TARGET_STORE_DIR` (Helm `config.rustfs.target_store_dir`) relocates both default queues on any platform. Linux and macOS behavior is unchanged when it is unset, persisted configuration is unchanged, and pending entries are not migrated when the root changes. See `docs/operations/target-queue-storage.md`.
|
|
||||||
|
|
||||||
### Deployment
|
### Deployment
|
||||||
|
|
||||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||||
|
|||||||
Generated
+4
@@ -9837,6 +9837,7 @@ dependencies = [
|
|||||||
"google-cloud-auth",
|
"google-cloud-auth",
|
||||||
"google-cloud-storage",
|
"google-cloud-storage",
|
||||||
"h2",
|
"h2",
|
||||||
|
"hashbrown 0.17.1",
|
||||||
"hex-simd",
|
"hex-simd",
|
||||||
"hmac 0.13.0",
|
"hmac 0.13.0",
|
||||||
"hostname",
|
"hostname",
|
||||||
@@ -10759,17 +10760,20 @@ dependencies = [
|
|||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-compression",
|
"async-compression",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"base64-simd",
|
||||||
"brotli 9.1.1",
|
"brotli 9.1.1",
|
||||||
"bytes",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"crc-fast",
|
"crc-fast",
|
||||||
"datafusion",
|
"datafusion",
|
||||||
|
"flatbuffers",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"hotpath",
|
"hotpath",
|
||||||
"http 1.5.0",
|
"http 1.5.0",
|
||||||
"lz4",
|
"lz4",
|
||||||
|
"lz4_flex",
|
||||||
"memchr",
|
"memchr",
|
||||||
"metrics",
|
"metrics",
|
||||||
"parking_lot 0.13.0",
|
"parking_lot 0.13.0",
|
||||||
|
|||||||
@@ -296,6 +296,7 @@ local-ip-address = "0.6.13"
|
|||||||
log = "0.4"
|
log = "0.4"
|
||||||
memmap2 = "0.9.12"
|
memmap2 = "0.9.12"
|
||||||
lz4 = "1.28.1"
|
lz4 = "1.28.1"
|
||||||
|
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||||
matchit = "0.9.2"
|
matchit = "0.9.2"
|
||||||
md-5 = "0.11.0"
|
md-5 = "0.11.0"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
|
|||||||
@@ -199,7 +199,7 @@ docker run -d --name rustfs -p 9000:9000 \
|
|||||||
Notes:
|
Notes:
|
||||||
- `RUSTFS_NOTIFY_ENABLE=true` enables the global notify module switch.
|
- `RUSTFS_NOTIFY_ENABLE=true` enables the global notify module switch.
|
||||||
- For ARN `arn:rustfs:sqs::primary:webhook`, use instance-scoped env vars with `_PRIMARY`.
|
- For ARN `arn:rustfs:sqs::primary:webhook`, use instance-scoped env vars with `_PRIMARY`.
|
||||||
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout, or set `RUSTFS_TARGET_STORE_DIR` to move every default queue; native Windows defaults to `%ProgramData%\RustFS`. See [Target queue storage](docs/operations/target-queue-storage.md).
|
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout.
|
||||||
- `RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY_PRIMARY` defaults to `false`; enabling it skips webhook TLS certificate verification, allows MITM attacks, and emits a startup warning. Prefer `RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA_PRIMARY` for private CAs.
|
- `RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY_PRIMARY` defaults to `false`; enabling it skips webhook TLS certificate verification, allows MITM attacks, and emits a startup warning. Prefer `RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA_PRIMARY` for private CAs.
|
||||||
- Since `1.0.0-beta.11`, webhook endpoints on private or container networks
|
- Since `1.0.0-beta.11`, webhook endpoints on private or container networks
|
||||||
(`Docker Compose service names`, `host.docker.internal`, RFC 1918 addresses) are
|
(`Docker Compose service names`, `host.docker.internal`, RFC 1918 addresses) are
|
||||||
|
|||||||
@@ -276,13 +276,6 @@ Drive timeout profile preset:
|
|||||||
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
|
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
|
||||||
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
|
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
|
||||||
|
|
||||||
## Target queue directory
|
|
||||||
|
|
||||||
- `RUSTFS_TARGET_STORE_DIR`
|
|
||||||
- absolute root for notify and audit target queues whose `queue_dir` is the default `/opt/rustfs/events` or `/opt/rustfs/audit`; they open `<root>/events` and `<root>/audit` on this node.
|
|
||||||
- unset keeps those paths on Linux and macOS; native Windows uses `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit`.
|
|
||||||
- explicit `queue_dir` values are never relocated, and the persisted target configuration is unchanged. Changing the root does not migrate pending entries. See `docs/operations/target-queue-storage.md`.
|
|
||||||
|
|
||||||
## Startup filesystem boundary policy
|
## Startup filesystem boundary policy
|
||||||
|
|
||||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||||
|
|||||||
@@ -184,15 +184,27 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
|||||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||||
/// one-time consumption of authenticated RPC signatures.
|
/// one-time consumption of authenticated RPC signatures.
|
||||||
///
|
///
|
||||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
/// Nonces remain protected until their signed timestamp is outside the observed wall-clock
|
||||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
/// validity window, including after clock rollback. Current-time signatures live about five
|
||||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
|
/// minutes; the maximum accepted forward clock skew can extend that to ten minutes. A body-bound
|
||||||
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
|
/// v3 request consumes both its v3 and compatibility-v2 nonces, so size for receiver nonce intake,
|
||||||
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
/// not S3 request rate or CPU count.
|
||||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
///
|
||||||
/// the shared secret) — and increments
|
/// Automatic admission budgets 13% of effective process memory for signed-time
|
||||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
/// hash tables, allocator overhead and simultaneous old/new tables during growth.
|
||||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
/// The reported count is an absolute UUID payload ceiling, not guaranteed throughput:
|
||||||
|
/// actual byte admission can reject earlier depending on timestamp distribution.
|
||||||
|
/// Tables grow with demand and are released together when their signed time expires.
|
||||||
|
/// CPU count, a historical minimum, and a fixed 32M ceiling do not constrain this budget.
|
||||||
|
/// If memory detection fails, the historical default remains a bounded count fallback.
|
||||||
|
///
|
||||||
|
/// Explicit values remain count ceilings with the historical minimum of 1,048,576
|
||||||
|
/// for configuration compatibility, and bypass automatic byte admission. That
|
||||||
|
/// compatibility floor does not apply to the automatically detected memory budget.
|
||||||
|
/// Zero never disables replay protection.
|
||||||
|
/// Overflow rejects the request without evicting a still-valid identity, increments
|
||||||
|
/// `rustfs_system_network_internode_replay_cache_overflow_total`, and returns resource
|
||||||
|
/// exhaustion rather than invalid credentials. Measure headroom before overriding.
|
||||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||||
|
|
||||||
|
|||||||
@@ -144,12 +144,6 @@ pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit";
|
|||||||
/// Applies to both notify and audit target queue stores.
|
/// Applies to both notify and audit target queue stores.
|
||||||
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||||
|
|
||||||
/// Environment variable that relocates the root of the default notify and audit queue directories.
|
|
||||||
///
|
|
||||||
/// It applies only to targets whose `queue_dir` is the logical default (`EVENT_DEFAULT_DIR` or
|
|
||||||
/// `AUDIT_DEFAULT_DIR`); their queues move to `<root>/events` and `<root>/audit` on this node.
|
|
||||||
pub const ENV_TARGET_STORE_DIR: &str = "RUSTFS_TARGET_STORE_DIR";
|
|
||||||
|
|
||||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||||
|
|
||||||
|
|||||||
@@ -121,6 +121,7 @@ test-util = []
|
|||||||
e2e-test-hooks = []
|
e2e-test-hooks = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
hashbrown.workspace = true
|
||||||
hotpath.workspace = true
|
hotpath.workspace = true
|
||||||
rustfs-filemeta.workspace = true
|
rustfs-filemeta.workspace = true
|
||||||
rustfs-utils = { workspace = true, features = ["full"] }
|
rustfs-utils = { workspace = true, features = ["full"] }
|
||||||
|
|||||||
@@ -640,17 +640,17 @@ pub mod rpc {
|
|||||||
pub use crate::cluster::rpc::{
|
pub use crate::cluster::rpc::{
|
||||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||||
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG,
|
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, RpcReplayCacheAdmissionError, S3PeerSys,
|
||||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity,
|
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement,
|
||||||
ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
|
ScannerPeerActivity, ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
|
||||||
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
|
ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||||
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
|
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
|
||||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
|
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||||
|
|||||||
@@ -229,6 +229,7 @@ pub struct ListPathRawOptions {
|
|||||||
pub hidden_entry_budget: usize,
|
pub hidden_entry_budget: usize,
|
||||||
pub pending_prefix: Option<String>,
|
pub pending_prefix: Option<String>,
|
||||||
pub filter_prefix: Option<String>,
|
pub filter_prefix: Option<String>,
|
||||||
|
pub identifier_tree_leaf: Option<String>,
|
||||||
pub forward_to: Option<String>,
|
pub forward_to: Option<String>,
|
||||||
pub min_disks: usize,
|
pub min_disks: usize,
|
||||||
/// Deliver every replica to the partial callback, including matching headers.
|
/// Deliver every replica to the partial callback, including matching headers.
|
||||||
@@ -268,6 +269,7 @@ impl Clone for ListPathRawOptions {
|
|||||||
hidden_entry_budget: self.hidden_entry_budget,
|
hidden_entry_budget: self.hidden_entry_budget,
|
||||||
pending_prefix: self.pending_prefix.clone(),
|
pending_prefix: self.pending_prefix.clone(),
|
||||||
filter_prefix: self.filter_prefix.clone(),
|
filter_prefix: self.filter_prefix.clone(),
|
||||||
|
identifier_tree_leaf: self.identifier_tree_leaf.clone(),
|
||||||
forward_to: self.forward_to.clone(),
|
forward_to: self.forward_to.clone(),
|
||||||
min_disks: self.min_disks,
|
min_disks: self.min_disks,
|
||||||
preserve_replica_metadata: self.preserve_replica_metadata,
|
preserve_replica_metadata: self.preserve_replica_metadata,
|
||||||
@@ -300,6 +302,7 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
|||||||
pending_prefix: opts.pending_prefix.clone(),
|
pending_prefix: opts.pending_prefix.clone(),
|
||||||
report_notfound: opts.report_not_found,
|
report_notfound: opts.report_not_found,
|
||||||
filter_prefix: opts.filter_prefix.clone(),
|
filter_prefix: opts.filter_prefix.clone(),
|
||||||
|
identifier_tree_leaf: opts.identifier_tree_leaf.clone(),
|
||||||
forward_to: opts.forward_to.clone(),
|
forward_to: opts.forward_to.clone(),
|
||||||
limit: opts.per_disk_limit,
|
limit: opts.per_disk_limit,
|
||||||
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
skip_total_timeout: opts.skip_walkdir_total_timeout,
|
||||||
@@ -1233,6 +1236,21 @@ mod tests {
|
|||||||
assert!(!options.skip_total_timeout);
|
assert!(!options.skip_total_timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn identifier_tree_selection_survives_job_clone_and_disk_request() {
|
||||||
|
let request = ListPathRawOptions {
|
||||||
|
path: "catalog/namespaces/".to_string(),
|
||||||
|
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||||
|
recursive: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let cloned = request.clone();
|
||||||
|
let walk = walk_dir_options(&cloned);
|
||||||
|
assert_eq!(walk.identifier_tree_leaf, request.identifier_tree_leaf);
|
||||||
|
assert_eq!(walk.base_dir, request.path);
|
||||||
|
assert!(walk.recursive);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||||
let request = ListPathRawOptions {
|
let request = ListPathRawOptions {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -35,17 +35,17 @@ pub use client::{
|
|||||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub use http_auth::{
|
pub use http_auth::{
|
||||||
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
|
RPC_BACKGROUND_ADMISSION_HEADER, RpcReplayCacheAdmissionError, TONIC_RPC_PREFIX, build_auth_headers,
|
||||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||||
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
|
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||||
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
|
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
|
||||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use crate::cluster::rpc::client::{
|
|||||||
node_service_time_out_client,
|
node_service_time_out_client,
|
||||||
};
|
};
|
||||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||||
use crate::core::pools::{PoolMeta, PoolMetaWriteState};
|
use crate::core::pools::{PoolMeta, PoolMetaSaveGate, PoolMetaSaveGateWriteGuard};
|
||||||
use crate::disk::error::DiskError;
|
use crate::disk::error::DiskError;
|
||||||
use crate::disk::error::{Error, Result};
|
use crate::disk::error::{Error, Result};
|
||||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||||
@@ -798,12 +798,12 @@ fn try_acquire_bucket_heal_movement_guard<'a>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn acquire_bucket_heal_write_guard<'a>(
|
async fn acquire_bucket_heal_write_guard<'a>(
|
||||||
gate: Option<&'a tokio::sync::Mutex<PoolMetaWriteState>>,
|
gate: Option<&'a PoolMetaSaveGate>,
|
||||||
) -> Result<Option<tokio::sync::MutexGuard<'a, PoolMetaWriteState>>> {
|
) -> Result<Option<PoolMetaSaveGateWriteGuard<'a>>> {
|
||||||
let Some(gate) = gate else {
|
let Some(gate) = gate else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let guard = gate.lock().await;
|
let guard = gate.write().await;
|
||||||
guard
|
guard
|
||||||
.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")
|
.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")
|
||||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(Error::other))?;
|
.map_err(|e| e.narrow_to_disk().unwrap_or_else(Error::other))?;
|
||||||
@@ -2388,8 +2388,8 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn local_bucket_heal_refuses_receiver_with_unsafe_pool_metadata() {
|
async fn local_bucket_heal_refuses_receiver_with_unsafe_pool_metadata() {
|
||||||
let gate = tokio::sync::Mutex::new(PoolMetaWriteState::default());
|
let gate = PoolMetaSaveGate::default();
|
||||||
gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
gate.write().await.observe_replicas(PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
repair_write_safe: false,
|
repair_write_safe: false,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -628,7 +628,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
|||||||
let fork = identity_epoch_fork(&fixture).await;
|
let fork = identity_epoch_fork(&fixture).await;
|
||||||
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed original failure timestamp should be valid");
|
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed original failure timestamp should be valid");
|
||||||
{
|
{
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||||
state.write_blocked = false;
|
state.write_blocked = false;
|
||||||
state.block_context = None;
|
state.block_context = None;
|
||||||
@@ -685,7 +685,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
|||||||
);
|
);
|
||||||
assert_committed(&fixture).await;
|
assert_committed(&fixture).await;
|
||||||
{
|
{
|
||||||
let state = fixture.store.pool_meta_save_gate.lock().await;
|
let state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
assert!(
|
assert!(
|
||||||
state.write_blocked,
|
state.write_blocked,
|
||||||
"candidate validation must propagate its permanent latch to the original state"
|
"candidate validation must propagate its permanent latch to the original state"
|
||||||
@@ -706,7 +706,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
|
|||||||
.expect("permanently blocked recovery should return without mutation"),
|
.expect("permanently blocked recovery should return without mutation"),
|
||||||
"ordinary-readable repaired heads must not clear the original known-fork latch"
|
"ordinary-readable repaired heads must not clear the original known-fork latch"
|
||||||
);
|
);
|
||||||
let state = fixture.store.pool_meta_save_gate.lock().await;
|
let state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
assert!(state.write_blocked);
|
assert!(state.write_blocked);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
state
|
state
|
||||||
@@ -979,7 +979,7 @@ async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_ra
|
|||||||
async {
|
async {
|
||||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(30),
|
std::time::Duration::from_secs(30),
|
||||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||||
@@ -1036,7 +1036,7 @@ async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without
|
|||||||
let before = snapshots(&fixture.store).await;
|
let before = snapshots(&fixture.store).await;
|
||||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(30),
|
std::time::Duration::from_secs(30),
|
||||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||||
@@ -1110,7 +1110,7 @@ async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
|||||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||||
.await.expect("mixed legacy heal must terminate")
|
.await.expect("mixed legacy heal must terminate")
|
||||||
@@ -1199,7 +1199,7 @@ async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order()
|
|||||||
let before = snapshots(&fixture.store).await;
|
let before = snapshots(&fixture.store).await;
|
||||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(30),
|
std::time::Duration::from_secs(30),
|
||||||
fixture
|
fixture
|
||||||
@@ -1301,7 +1301,7 @@ async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes()
|
|||||||
let before = snapshots(&fixture.store).await;
|
let before = snapshots(&fixture.store).await;
|
||||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(30),
|
std::time::Duration::from_secs(30),
|
||||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||||
@@ -1410,7 +1410,7 @@ async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes()
|
|||||||
let before = snapshots(&fixture.store).await;
|
let before = snapshots(&fixture.store).await;
|
||||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||||
let guard = owner(&fixture.store).await;
|
let guard = owner(&fixture.store).await;
|
||||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
let mut state = fixture.store.pool_meta_save_gate.write().await;
|
||||||
let result = tokio::time::timeout(
|
let result = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(30),
|
std::time::Duration::from_secs(30),
|
||||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||||
|
|||||||
+431
-124
@@ -6759,22 +6759,129 @@ impl PoolMetaWriteState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
|
pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
|
||||||
if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
|
let (Some(mut context), _) = self.write_block() else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
};
|
||||||
let mut context = self
|
|
||||||
.block_context
|
|
||||||
.clone()
|
|
||||||
.or_else(|| {
|
|
||||||
self.transaction_failure
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
||||||
.clone()
|
|
||||||
})
|
|
||||||
.unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
|
|
||||||
context.operation = operation.to_owned();
|
context.operation = operation.to_owned();
|
||||||
Err(Error::other(context))
|
Err(Error::other(context))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the context blocking writes and whether the shared transaction
|
||||||
|
/// latch was set in the same observation.
|
||||||
|
///
|
||||||
|
/// Outcomes latch, and recovery clears, the shared abort flag together with
|
||||||
|
/// its failure under the failure lock. Re-reading the flag under that lock
|
||||||
|
/// gives inspection that does not hold the gate one consistent transition.
|
||||||
|
fn write_block(&self) -> (Option<crate::error::PoolMetadataError>, bool) {
|
||||||
|
if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
|
||||||
|
return (None, false);
|
||||||
|
}
|
||||||
|
let failure = self
|
||||||
|
.transaction_failure
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
let aborted = self.aborted_transaction.load(Ordering::SeqCst);
|
||||||
|
if !self.write_blocked && !aborted {
|
||||||
|
return (None, false);
|
||||||
|
}
|
||||||
|
let context = self
|
||||||
|
.block_context
|
||||||
|
.clone()
|
||||||
|
.or_else(|| failure.clone())
|
||||||
|
.unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
|
||||||
|
(Some(context), aborted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Local pool metadata authority gate.
|
||||||
|
///
|
||||||
|
/// Readers hold an immutable admission proof. Writers serialize publication,
|
||||||
|
/// recovery and authority changes. Every writer publishes the state it leaves
|
||||||
|
/// while it still owns the gate, so inspection can read the last committed
|
||||||
|
/// authority without queueing behind metadata I/O. The published copy shares
|
||||||
|
/// the transaction latches with the live state, so an outcome that latches a
|
||||||
|
/// failure after its writer released the gate is visible immediately.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct PoolMetaSaveGate {
|
||||||
|
state: tokio::sync::RwLock<PoolMetaWriteState>,
|
||||||
|
published: arc_swap::ArcSwap<PoolMetaWriteState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PoolMetaSaveGate {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(PoolMetaWriteState::default())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PoolMetaSaveGate {
|
||||||
|
pub(crate) fn new(state: PoolMetaWriteState) -> Self {
|
||||||
|
Self {
|
||||||
|
published: arc_swap::ArcSwap::from_pointee(state.clone()),
|
||||||
|
state: tokio::sync::RwLock::new(state),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, PoolMetaWriteState> {
|
||||||
|
self.state.read().await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn try_read(
|
||||||
|
&self,
|
||||||
|
) -> std::result::Result<tokio::sync::RwLockReadGuard<'_, PoolMetaWriteState>, tokio::sync::TryLockError> {
|
||||||
|
self.state.try_read()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn write(&self) -> PoolMetaSaveGateWriteGuard<'_> {
|
||||||
|
PoolMetaSaveGateWriteGuard {
|
||||||
|
state: self.state.write().await,
|
||||||
|
published: &self.published,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn try_write(&self) -> std::result::Result<PoolMetaSaveGateWriteGuard<'_>, tokio::sync::TryLockError> {
|
||||||
|
self.state.try_write().map(|state| PoolMetaSaveGateWriteGuard {
|
||||||
|
state,
|
||||||
|
published: &self.published,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the authority left by the most recent writer without waiting.
|
||||||
|
///
|
||||||
|
/// Changes an active writer makes become visible when it releases the
|
||||||
|
/// gate; shared transaction latches are read live.
|
||||||
|
pub(crate) fn published(&self) -> Arc<PoolMetaWriteState> {
|
||||||
|
self.published.load_full()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exclusive authority ownership that publishes its final state on release.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct PoolMetaSaveGateWriteGuard<'a> {
|
||||||
|
state: tokio::sync::RwLockWriteGuard<'a, PoolMetaWriteState>,
|
||||||
|
published: &'a arc_swap::ArcSwap<PoolMetaWriteState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::ops::Deref for PoolMetaSaveGateWriteGuard<'_> {
|
||||||
|
type Target = PoolMetaWriteState;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.state
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::ops::DerefMut for PoolMetaSaveGateWriteGuard<'_> {
|
||||||
|
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||||
|
&mut self.state
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for PoolMetaSaveGateWriteGuard<'_> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
// The tokio guard is released after this body, so publications follow
|
||||||
|
// writer order and a later writer can never be overwritten by an
|
||||||
|
// earlier one.
|
||||||
|
self.published.store(Arc::new(self.state.clone()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pool_metadata_error(
|
fn pool_metadata_error(
|
||||||
@@ -13438,7 +13545,7 @@ impl ECStore {
|
|||||||
"Decommission cancel will retry target capacity fencing"
|
"Decommission cancel will retry target capacity fencing"
|
||||||
);
|
);
|
||||||
tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
|
tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (_read_guard, snapshot) = self
|
let (_read_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
|
.acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13482,7 +13589,7 @@ impl ECStore {
|
|||||||
target_pool_indices: &[usize],
|
target_pool_indices: &[usize],
|
||||||
admission: DecommissionCapacityAdmission,
|
admission: DecommissionCapacityAdmission,
|
||||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (pool_meta_guard, snapshot) = self
|
let (pool_meta_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13514,12 +13621,8 @@ impl ECStore {
|
|||||||
pub(crate) async fn acquire_pool_meta_object_heal_fence(
|
pub(crate) async fn acquire_pool_meta_object_heal_fence(
|
||||||
&self,
|
&self,
|
||||||
target_pool_indices: &[usize],
|
target_pool_indices: &[usize],
|
||||||
) -> Result<(
|
) -> Result<(Arc<rustfs_lock::NamespaceLockGuard>, Vec<Result<()>>, PoolMetaSaveGateWriteGuard<'_>)> {
|
||||||
Arc<rustfs_lock::NamespaceLockGuard>,
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
Vec<Result<()>>,
|
|
||||||
tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
|
|
||||||
)> {
|
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
|
||||||
let (pool_meta_guard, snapshot) = self
|
let (pool_meta_guard, snapshot) = self
|
||||||
.acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
|
.acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13539,7 +13642,7 @@ impl ECStore {
|
|||||||
pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
|
pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
|
||||||
&self,
|
&self,
|
||||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (pool_meta_guard, snapshot) = self
|
let (pool_meta_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
|
.acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13606,7 +13709,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let (reconciliations, model_version) = {
|
let (reconciliations, model_version) = {
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (_read_guard, snapshot) = self
|
let (_read_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
|
.acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13623,7 +13726,7 @@ impl ECStore {
|
|||||||
let mut legacy_write_fence = None;
|
let mut legacy_write_fence = None;
|
||||||
match model_version {
|
match model_version {
|
||||||
Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
|
Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (write_guard, snapshot) = self
|
let (write_guard, snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13698,7 +13801,7 @@ impl ECStore {
|
|||||||
let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
|
let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
|
||||||
fence
|
fence
|
||||||
} else {
|
} else {
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (write_guard, snapshot) = self
|
let (write_guard, snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -13786,7 +13889,7 @@ impl ECStore {
|
|||||||
if let Some(target_guard) = target_guard.as_ref() {
|
if let Some(target_guard) = target_guard.as_ref() {
|
||||||
ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
|
ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
|
||||||
}
|
}
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -14009,7 +14112,7 @@ impl ECStore {
|
|||||||
"equivalent temporary target reconciliation",
|
"equivalent temporary target reconciliation",
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -14204,7 +14307,7 @@ impl ECStore {
|
|||||||
let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
|
let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
|
||||||
let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
|
let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
|
||||||
let mut operation = Some(operation);
|
let mut operation = Some(operation);
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (read_guard, snapshot) = self
|
let (read_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -14270,7 +14373,7 @@ impl ECStore {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pause_decommission_capacity_before_owner_write(self.id).await;
|
pause_decommission_capacity_before_owner_write(self.id).await;
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (write_guard, mut snapshot) = self
|
let (write_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -14430,7 +14533,7 @@ impl ECStore {
|
|||||||
let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
|
let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
|
||||||
if let Some(target_guard) = target_guard.as_ref() {
|
if let Some(target_guard) = target_guard.as_ref() {
|
||||||
ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
|
ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
|
||||||
let mut finalize_save_guard = self.pool_meta_save_gate.lock().await;
|
let mut finalize_save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (finalize_write_guard, finalize_snapshot) = self
|
let (finalize_write_guard, finalize_snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
|
.acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -14592,7 +14695,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
|
pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
|
||||||
self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
|
self.pool_meta_save_gate.read().await.ensure_write_safe(operation)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reports whether pool metadata side effects are currently writable.
|
/// Reports whether pool metadata side effects are currently writable.
|
||||||
@@ -14602,27 +14705,16 @@ impl ECStore {
|
|||||||
self.pool_meta_write_status().await.is_ok()
|
self.pool_meta_write_status().await.is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A health probe must not wait behind a metadata transaction's disk I/O.
|
/// Inspects the authority committed by the most recent writer. Health must
|
||||||
|
/// not queue behind metadata transaction I/O or behind a writer waiting for
|
||||||
|
/// readers, so this never acquires the gate.
|
||||||
pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
|
pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
|
||||||
let Ok(write_state) = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock()).await
|
let write_state = self.pool_meta_save_gate.published();
|
||||||
else {
|
let (block, transaction_aborted) = write_state.write_block();
|
||||||
return PoolMetaWriteGateStatus {
|
let writes_ready = block.is_none();
|
||||||
writes_ready: false,
|
let context = block.as_ref();
|
||||||
check_timed_out: true,
|
|
||||||
..PoolMetaWriteGateStatus::default()
|
|
||||||
};
|
|
||||||
};
|
|
||||||
let transaction_aborted = write_state.aborted_transaction.load(Ordering::SeqCst);
|
|
||||||
let writes_ready = !write_state.write_blocked && !transaction_aborted;
|
|
||||||
let failure = if writes_ready {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
write_state.ensure_write_safe("pool metadata snapshot").err()
|
|
||||||
};
|
|
||||||
let context = failure.as_ref().and_then(Error::pool_metadata_failure);
|
|
||||||
PoolMetaWriteGateStatus {
|
PoolMetaWriteGateStatus {
|
||||||
writes_ready,
|
writes_ready,
|
||||||
check_timed_out: false,
|
|
||||||
write_blocked: write_state.write_blocked,
|
write_blocked: write_state.write_blocked,
|
||||||
transaction_aborted,
|
transaction_aborted,
|
||||||
pool_meta_absent: write_state.pool_meta_absent,
|
pool_meta_absent: write_state.pool_meta_absent,
|
||||||
@@ -14635,12 +14727,13 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A health probe must not wait behind a metadata transaction's disk I/O.
|
/// Readiness view of the committed authority; see
|
||||||
|
/// [`Self::pool_meta_write_gate_status`]. Mutating paths must keep calling
|
||||||
|
/// `ensure_write_safe` under the gate instead of relying on this view.
|
||||||
pub async fn pool_meta_write_status(&self) -> Result<()> {
|
pub async fn pool_meta_write_status(&self) -> Result<()> {
|
||||||
let write_state = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock())
|
self.pool_meta_save_gate
|
||||||
.await
|
.published()
|
||||||
.map_err(|_| Error::Timeout)?;
|
.ensure_write_safe("pool metadata readiness")
|
||||||
write_state.ensure_write_safe("pool metadata readiness")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn heal_pool_meta_replica_under_fence(
|
pub(crate) async fn heal_pool_meta_replica_under_fence(
|
||||||
@@ -14707,7 +14800,7 @@ impl ECStore {
|
|||||||
|
|
||||||
pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
|
pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
|
||||||
{
|
{
|
||||||
let Ok(state) = self.pool_meta_save_gate.try_lock() else {
|
let Ok(state) = self.pool_meta_save_gate.try_read() else {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
};
|
};
|
||||||
if state.ensure_write_safe("pool metadata recovery").is_ok() {
|
if state.ensure_write_safe("pool metadata recovery").is_ok() {
|
||||||
@@ -14741,7 +14834,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||||
let _movement_guard = movement_gate.write().await;
|
let _movement_guard = movement_gate.write().await;
|
||||||
let mut state = self.pool_meta_save_gate.lock().await;
|
let mut state = self.pool_meta_save_gate.write().await;
|
||||||
// Outcomes can outlive the save-gate guard. Do not let an old Drop
|
// Outcomes can outlive the save-gate guard. Do not let an old Drop
|
||||||
// relatch, or an older recovery clear a newly installed block.
|
// relatch, or an older recovery clear a newly installed block.
|
||||||
if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
|
if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
|
||||||
@@ -14817,15 +14910,20 @@ impl ECStore {
|
|||||||
candidate.transaction_failure = Arc::clone(&state.transaction_failure);
|
candidate.transaction_failure = Arc::clone(&state.transaction_failure);
|
||||||
candidate.active_transactions = Arc::clone(&state.active_transactions);
|
candidate.active_transactions = Arc::clone(&state.active_transactions);
|
||||||
candidate.block_started_at = Arc::clone(&state.block_started_at);
|
candidate.block_started_at = Arc::clone(&state.block_started_at);
|
||||||
|
{
|
||||||
|
// Inspection reads these shared latches without the gate; clear the
|
||||||
|
// flag and its failure in one failure-lock section.
|
||||||
|
let mut failure = state
|
||||||
|
.transaction_failure
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
state.aborted_transaction.store(false, Ordering::SeqCst);
|
||||||
|
*failure = None;
|
||||||
|
}
|
||||||
*state
|
*state
|
||||||
.block_started_at
|
.block_started_at
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||||
*state
|
|
||||||
.transaction_failure
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
|
||||||
state.aborted_transaction.store(false, Ordering::SeqCst);
|
|
||||||
*state = candidate;
|
*state = candidate;
|
||||||
info!(
|
info!(
|
||||||
event = EVENT_DECOMMISSION_STATE,
|
event = EVENT_DECOMMISSION_STATE,
|
||||||
@@ -14841,7 +14939,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
|
pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
|
||||||
let Ok(mut state) = self.pool_meta_save_gate.try_lock() else {
|
let Ok(mut state) = self.pool_meta_save_gate.try_write() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if state.ensure_write_safe("pool metadata recovery").is_ok() {
|
if state.ensure_write_safe("pool metadata recovery").is_ok() {
|
||||||
@@ -14913,7 +15011,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
|
pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
|
||||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
let mut write_state = self.pool_meta_save_gate.write().await;
|
||||||
write_state.ensure_write_safe(operation)?;
|
write_state.ensure_write_safe(operation)?;
|
||||||
let pool = self
|
let pool = self
|
||||||
.pools
|
.pools
|
||||||
@@ -14976,7 +15074,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn save_current_pool_meta_checked(&self, indices: &[usize], running: Option<(usize, PoolStatus)>) -> Result<()> {
|
async fn save_current_pool_meta_checked(&self, indices: &[usize], running: Option<(usize, PoolStatus)>) -> Result<()> {
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15078,7 +15176,7 @@ impl ECStore {
|
|||||||
// Lock order: save gate, rebalance metadata, then the short pool
|
// Lock order: save gate, rebalance metadata, then the short pool
|
||||||
// metadata read/write sections. Peer reloads are intentionally
|
// metadata read/write sections. Peer reloads are intentionally
|
||||||
// performed by the caller after both locks are released.
|
// performed by the caller after both locks are released.
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15156,7 +15254,7 @@ impl ECStore {
|
|||||||
) -> Result<bool> {
|
) -> Result<bool> {
|
||||||
// Lock order: save gate -> pool metadata namespace -> rebalance metadata
|
// Lock order: save gate -> pool metadata namespace -> rebalance metadata
|
||||||
// -> short local pool metadata sections. Publish only after the save.
|
// -> short local pool metadata sections. Publish only after the save.
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15232,7 +15330,7 @@ impl ECStore {
|
|||||||
indices: &[usize],
|
indices: &[usize],
|
||||||
decom_buckets: Vec<DecomBucketInfo>,
|
decom_buckets: Vec<DecomBucketInfo>,
|
||||||
) -> Result<(PoolMeta, PoolMeta)> {
|
) -> Result<(PoolMeta, PoolMeta)> {
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
save_guard.ensure_write_safe("decommission start failed")?;
|
save_guard.ensure_write_safe("decommission start failed")?;
|
||||||
let rebalance_pool = self
|
let rebalance_pool = self
|
||||||
.pools
|
.pools
|
||||||
@@ -15377,7 +15475,7 @@ impl ECStore {
|
|||||||
indices: &[usize],
|
indices: &[usize],
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let _movement_guard = movement_gate.write().await;
|
let _movement_guard = movement_gate.write().await;
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15726,7 +15824,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
let _start_guard = self.start_gate.lock().await;
|
let _start_guard = self.start_gate.lock().await;
|
||||||
let plan = {
|
let plan = {
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (_read_guard, snapshot) = self
|
let (_read_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "decommission intent adoption planning failed")
|
.acquire_pool_meta_read_guard(&save_guard, "decommission intent adoption planning failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15745,7 +15843,7 @@ impl ECStore {
|
|||||||
target_guards.push((intent.target_pool_index, guard));
|
target_guards.push((intent.target_pool_index, guard));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission intent adoption failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission intent adoption failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15966,7 +16064,7 @@ impl ECStore {
|
|||||||
// global lock, then fence the exact target cohort before taking the
|
// global lock, then fence the exact target cohort before taking the
|
||||||
// write lock used to publish the terminal transition.
|
// write lock used to publish the terminal transition.
|
||||||
let terminal_fence_plan = if acquire_runtime_fence {
|
let terminal_fence_plan = if acquire_runtime_fence {
|
||||||
let read_save_guard = self.pool_meta_save_gate.lock().await;
|
let read_save_guard = self.pool_meta_save_gate.read().await;
|
||||||
let (read_guard, snapshot) = self
|
let (read_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
|
.acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -15983,7 +16081,7 @@ impl ECStore {
|
|||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
|
let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
|
||||||
let (guard, pool_meta) = self
|
let (guard, pool_meta) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
|
||||||
@@ -16422,7 +16520,7 @@ impl ECStore {
|
|||||||
let _start_guard = self.start_gate.lock().await;
|
let _start_guard = self.start_gate.lock().await;
|
||||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||||
let _movement_guard = movement_gate.write().await;
|
let _movement_guard = movement_gate.write().await;
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -16536,7 +16634,7 @@ impl ECStore {
|
|||||||
let _start_guard = self.start_gate.lock().await;
|
let _start_guard = self.start_gate.lock().await;
|
||||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||||
let _movement_guard = movement_gate.write().await;
|
let _movement_guard = movement_gate.write().await;
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
save_guard.ensure_write_safe("decommission convergence recheck failed")?;
|
save_guard.ensure_write_safe("decommission convergence recheck failed")?;
|
||||||
let pool = self
|
let pool = self
|
||||||
.pools
|
.pools
|
||||||
@@ -16630,7 +16728,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
|
async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
|
||||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
let mut save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, mut snapshot) = self
|
let (pool_meta_guard, mut snapshot) = self
|
||||||
.acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
|
.acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -16750,7 +16848,7 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let _start_guard = self.start_gate.lock().await;
|
let _start_guard = self.start_gate.lock().await;
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.write().await;
|
||||||
save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
|
save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
|
||||||
let indices = {
|
let indices = {
|
||||||
let pool_meta = self.pool_meta.read().await;
|
let pool_meta = self.pool_meta.read().await;
|
||||||
@@ -17726,7 +17824,7 @@ impl ECStore {
|
|||||||
.await?;
|
.await?;
|
||||||
// Native replica writers acquire this same fixed object domain before
|
// Native replica writers acquire this same fixed object domain before
|
||||||
// durable pool metadata. Keep both fences through physical source deletion.
|
// durable pool metadata. Keep both fences through physical source deletion.
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.write().await;
|
||||||
let (pool_meta_guard, snapshot) = self
|
let (pool_meta_guard, snapshot) = self
|
||||||
.acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
|
.acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -22189,7 +22287,7 @@ async fn persist_v3_pool_meta_for_test(store: &Arc<ECStore>) {
|
|||||||
.await
|
.await
|
||||||
.expect("the baseline pool metadata should be readable before V3 migration");
|
.expect("the baseline pool metadata should be readable before V3 migration");
|
||||||
meta.version = POOL_META_GENERATION_VERSION;
|
meta.version = POOL_META_GENERATION_VERSION;
|
||||||
let mut write_state = store.pool_meta_save_gate.lock().await.clone();
|
let mut write_state = store.pool_meta_save_gate.write().await.clone();
|
||||||
temp_env::async_with_vars(
|
temp_env::async_with_vars(
|
||||||
[
|
[
|
||||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
|
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
|
||||||
@@ -22294,7 +22392,7 @@ mod tests {
|
|||||||
assert_eq!(snapshot[1].diagnostic.availability, PoolAvailability::Offline);
|
assert_eq!(snapshot[1].diagnostic.availability, PoolAvailability::Offline);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.ensure_write_safe("ordinary object write")
|
.ensure_write_safe("ordinary object write")
|
||||||
.expect("a failed diagnostic read must not latch the writer gate");
|
.expect("a failed diagnostic read must not latch the writer gate");
|
||||||
@@ -23966,7 +24064,7 @@ mod tests {
|
|||||||
let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
|
let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
|
||||||
{
|
{
|
||||||
// Preserve the normal save-gate -> namespace-writer lock order.
|
// Preserve the normal save-gate -> namespace-writer lock order.
|
||||||
let mut write_state = store.pool_meta_save_gate.lock().await;
|
let mut write_state = store.pool_meta_save_gate.write().await;
|
||||||
let (guard, _) = store
|
let (guard, _) = store
|
||||||
.acquire_pool_meta_write_guard(&mut write_state, "V1 test baseline")
|
.acquire_pool_meta_write_guard(&mut write_state, "V1 test baseline")
|
||||||
.await
|
.await
|
||||||
@@ -25150,8 +25248,8 @@ mod tests {
|
|||||||
.expect("first legacy mutation should enter")
|
.expect("first legacy mutation should enter")
|
||||||
.expect("first legacy mutation should report entry");
|
.expect("first legacy mutation should report entry");
|
||||||
assert!(
|
assert!(
|
||||||
store.pool_meta_save_gate.try_lock().is_err(),
|
store.pool_meta_save_gate.try_read().is_err(),
|
||||||
"legacy target I/O must retain the local pool metadata save gate"
|
"legacy target I/O must retain exclusive ownership of the local pool metadata save gate"
|
||||||
);
|
);
|
||||||
let pool_meta_lock = store.pools[0]
|
let pool_meta_lock = store.pools[0]
|
||||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||||
@@ -25296,7 +25394,7 @@ mod tests {
|
|||||||
|
|
||||||
let save_guard = store
|
let save_guard = store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.try_lock()
|
.try_write()
|
||||||
.expect("target I/O must not retain the local pool metadata save gate");
|
.expect("target I/O must not retain the local pool metadata save gate");
|
||||||
drop(save_guard);
|
drop(save_guard);
|
||||||
let pool_meta_lock = store.pools[0]
|
let pool_meta_lock = store.pools[0]
|
||||||
@@ -25443,7 +25541,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("create single-pool bucket before blocking pool metadata writes");
|
.expect("create single-pool bucket before blocking pool metadata writes");
|
||||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
|
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
|
||||||
store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
|
store.pool_meta_save_gate.write().await.block_writes_after_fence_loss();
|
||||||
let write_gate = store.pool_meta_write_gate_status().await;
|
let write_gate = store.pool_meta_write_gate_status().await;
|
||||||
assert!(!write_gate.writes_ready);
|
assert!(!write_gate.writes_ready);
|
||||||
assert!(write_gate.write_blocked);
|
assert!(write_gate.write_blocked);
|
||||||
@@ -27642,7 +27740,7 @@ mod tests {
|
|||||||
let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
|
let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
|
||||||
disks.push((set.clone(), original));
|
disks.push((set.clone(), original));
|
||||||
}
|
}
|
||||||
let mut state = store.pool_meta_save_gate.lock().await;
|
let mut state = store.pool_meta_save_gate.write().await;
|
||||||
let err = store
|
let err = store
|
||||||
.acquire_pool_meta_write_guard(&mut state, "writer preflight")
|
.acquire_pool_meta_write_guard(&mut state, "writer preflight")
|
||||||
.await
|
.await
|
||||||
@@ -27667,7 +27765,7 @@ mod tests {
|
|||||||
requested.pools[0].last_update += Duration::seconds(1);
|
requested.pools[0].last_update += Duration::seconds(1);
|
||||||
requested.version = POOL_META_GENERATION_VERSION;
|
requested.version = POOL_META_GENERATION_VERSION;
|
||||||
{
|
{
|
||||||
let mut state = store.pool_meta_save_gate.lock().await;
|
let mut state = store.pool_meta_save_gate.write().await;
|
||||||
let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
|
let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
|
||||||
let outcome = requested
|
let outcome = requested
|
||||||
.save_no_lock_armed(store.pools.clone(), &mut state, PoolMetaPersistenceFence::Owned(Arc::clone(&fence)), &[0])
|
.save_no_lock_armed(store.pools.clone(), &mut state, PoolMetaPersistenceFence::Owned(Arc::clone(&fence)), &[0])
|
||||||
@@ -27685,42 +27783,253 @@ mod tests {
|
|||||||
assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
|
assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
|
||||||
worker.release();
|
worker.release();
|
||||||
store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
|
store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
|
||||||
// An outcome may outlive the save mutex. Its Drop must precede recovery.
|
// An outcome may outlive the authority writer. Its Drop must precede recovery.
|
||||||
let outstanding = store.pool_meta_save_gate.lock().await.arm_transaction();
|
let outstanding = store.pool_meta_save_gate.write().await.arm_transaction();
|
||||||
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
||||||
drop(outstanding);
|
drop(outstanding);
|
||||||
assert!(store.recover_pool_meta_transaction().await.unwrap());
|
let authority_reader = store.pool_meta_save_gate.read().await;
|
||||||
|
let recovery = tokio::spawn({
|
||||||
|
let store = store.clone();
|
||||||
|
async move { store.recover_pool_meta_transaction().await }
|
||||||
|
});
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
while store.pool_meta_save_gate.try_read().is_ok() {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("recovery must queue for its authority writer while a reader remains");
|
||||||
|
drop(authority_reader);
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), recovery)
|
||||||
|
.await
|
||||||
|
.expect("the queued recovery writer must make progress")
|
||||||
|
.expect("the recovery task must not panic")
|
||||||
|
.expect("the durable metadata must recover")
|
||||||
|
);
|
||||||
assert!(store.pool_meta_writes_ready().await);
|
assert!(store.pool_meta_writes_ready().await);
|
||||||
let recovered = store.pool_meta_write_gate_status().await;
|
let recovered = store.pool_meta_write_gate_status().await;
|
||||||
assert!(recovered.writes_ready);
|
assert!(recovered.writes_ready);
|
||||||
assert!(!recovered.check_timed_out);
|
|
||||||
assert!(!recovered.write_blocked);
|
assert!(!recovered.write_blocked);
|
||||||
assert!(!recovered.transaction_aborted);
|
assert!(!recovered.transaction_aborted);
|
||||||
assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
|
assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
|
||||||
assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
|
assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
|
||||||
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
||||||
store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
|
store.pool_meta_save_gate.write().await.block_writes_after_fence_loss();
|
||||||
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
assert!(!store.recover_pool_meta_transaction().await.unwrap());
|
||||||
assert!(!store.pool_meta_writes_ready().await);
|
assert!(!store.pool_meta_writes_ready().await);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
async fn pool_meta_readiness_is_bounded_while_save_gate_is_held() {
|
async fn pool_meta_readiness_does_not_queue_behind_read_only_fence() {
|
||||||
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||||
let _state = store.pool_meta_save_gate.lock().await;
|
let namespace = store.pools[0]
|
||||||
let result = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
|
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.expect("create the pool metadata namespace lock");
|
||||||
assert!(matches!(result, Err(Error::Timeout)));
|
let writer = namespace
|
||||||
|
.get_write_lock(get_lock_acquire_timeout())
|
||||||
|
.await
|
||||||
|
.expect("hold the namespace writer while the read-only admission waits");
|
||||||
|
let admission = tokio::spawn({
|
||||||
|
let store = store.clone();
|
||||||
|
async move { store.acquire_decommission_capacity_release_fence_with_active_source().await }
|
||||||
|
});
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
if store.pool_meta_save_gate.try_write().is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the read-only admission must reach its authority fence");
|
||||||
|
|
||||||
|
let readiness = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
|
||||||
|
.await
|
||||||
|
.expect("readiness inspection must not wait for the authority gate");
|
||||||
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
|
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.expect("the authority snapshot must not wait for the authority gate");
|
||||||
assert!(!snapshot.writes_ready);
|
assert!(
|
||||||
assert!(snapshot.check_timed_out);
|
store.pool_meta_save_gate.try_write().is_err() && !admission.is_finished(),
|
||||||
|
"the read-only admission must still hold its authority reader while health is sampled"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A local writer queued behind the parked reader blocks every later
|
||||||
|
// gate reader. Inspection must still answer from committed authority.
|
||||||
|
let queued_writer = tokio::spawn({
|
||||||
|
let store = store.clone();
|
||||||
|
async move { store.save_current_pool_meta_for_test(&[0]).await }
|
||||||
|
});
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
while store.pool_meta_save_gate.try_read().is_ok() {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the local writer must queue behind the parked reader");
|
||||||
|
let queued_readiness = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
|
||||||
|
.await
|
||||||
|
.expect("readiness inspection must not queue behind a waiting writer");
|
||||||
|
let queued_snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
|
||||||
|
.await
|
||||||
|
.expect("the authority snapshot must not queue behind a waiting writer");
|
||||||
|
assert!(
|
||||||
|
!admission.is_finished() && !queued_writer.is_finished(),
|
||||||
|
"the reader must still be parked and the writer still queued while health is sampled"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(writer);
|
||||||
|
let (read_fence, _) = tokio::time::timeout(std::time::Duration::from_secs(10), admission)
|
||||||
|
.await
|
||||||
|
.expect("the read-only admission must finish after releasing the namespace writer")
|
||||||
|
.expect("the admission task must not panic")
|
||||||
|
.expect("the durable metadata must still admit its ordinary read fence");
|
||||||
|
drop(read_fence);
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), queued_writer)
|
||||||
|
.await
|
||||||
|
.expect("the queued writer must make progress after the reader releases")
|
||||||
|
.expect("the queued writer task must not panic")
|
||||||
|
.expect("the queued pool metadata save must commit");
|
||||||
|
|
||||||
|
readiness.expect("a read-only namespace wait must not hide unchanged writable authority");
|
||||||
|
assert!(snapshot.writes_ready, "read-only I/O must not make authority unavailable: {snapshot:?}");
|
||||||
|
queued_readiness.expect("a queued writer must not hide unchanged writable authority");
|
||||||
|
assert!(
|
||||||
|
queued_snapshot.writes_ready,
|
||||||
|
"a queued writer must not make authority unavailable: {queued_snapshot:?}"
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.pool_meta_write_status()
|
||||||
|
.await
|
||||||
|
.expect("the committed save must leave authority writable");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn pool_meta_readiness_reports_committed_authority_while_save_gate_is_held() {
|
||||||
|
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||||
|
let mut state = store.pool_meta_save_gate.write().await;
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
|
||||||
|
.await
|
||||||
|
.expect("readiness inspection must not wait for an active writer")
|
||||||
|
.expect("an active writer must not hide committed writable authority");
|
||||||
|
let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
|
||||||
|
.await
|
||||||
|
.expect("the authority snapshot must not wait for an active writer");
|
||||||
|
assert!(snapshot.writes_ready, "{snapshot:?}");
|
||||||
assert!(!snapshot.write_blocked);
|
assert!(!snapshot.write_blocked);
|
||||||
assert!(!snapshot.transaction_aborted);
|
assert!(!snapshot.transaction_aborted);
|
||||||
assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
|
assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
|
||||||
|
|
||||||
|
// A latch installed by the writer commits when it releases the gate.
|
||||||
|
state.block_writes_after_fence_loss();
|
||||||
|
assert!(
|
||||||
|
store.pool_meta_write_gate_status().await.writes_ready,
|
||||||
|
"inspection reports the committed authority until the writer releases the gate"
|
||||||
|
);
|
||||||
|
drop(state);
|
||||||
|
let blocked = store.pool_meta_write_gate_status().await;
|
||||||
|
assert!(!blocked.writes_ready, "{blocked:?}");
|
||||||
|
assert!(blocked.write_blocked);
|
||||||
|
assert_eq!((blocked.reason, blocked.phase), (Some("fence_lost"), Some("format_heal")));
|
||||||
|
let err = store
|
||||||
|
.pool_meta_write_status()
|
||||||
|
.await
|
||||||
|
.expect_err("a released writer latch must fail readiness closed");
|
||||||
|
assert_eq!(
|
||||||
|
err.pool_metadata_failure()
|
||||||
|
.expect("the readiness failure must carry pool metadata context")
|
||||||
|
.kind,
|
||||||
|
crate::error::PoolMetadataFailure::FenceLost
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn pool_meta_readiness_observes_outcome_latch_while_save_gate_is_held() {
|
||||||
|
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||||
|
let arm = {
|
||||||
|
let state = store.pool_meta_save_gate.write().await;
|
||||||
|
let mut arm = state.arm_transaction();
|
||||||
|
arm.phase = Some("publication");
|
||||||
|
arm
|
||||||
|
};
|
||||||
|
// The outcome outlives its writer while another writer owns the gate.
|
||||||
|
let writer = store.pool_meta_save_gate.write().await;
|
||||||
|
drop(arm);
|
||||||
|
let status = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
|
||||||
|
.await
|
||||||
|
.expect("the authority snapshot must not wait for an active writer");
|
||||||
|
assert!(!status.writes_ready, "{status:?}");
|
||||||
|
assert!(status.transaction_aborted);
|
||||||
|
assert!(!status.write_blocked);
|
||||||
|
assert_eq!((status.reason, status.phase), (Some("transaction_unknown"), Some("publication")));
|
||||||
|
let err = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
|
||||||
|
.await
|
||||||
|
.expect("readiness inspection must not wait for an active writer")
|
||||||
|
.expect_err("a live outcome latch must fail readiness closed while a writer is active");
|
||||||
|
assert_eq!(
|
||||||
|
err.pool_metadata_failure()
|
||||||
|
.expect("the readiness failure must carry pool metadata context")
|
||||||
|
.kind,
|
||||||
|
crate::error::PoolMetadataFailure::TransactionUnknown
|
||||||
|
);
|
||||||
|
drop(writer);
|
||||||
|
assert!(
|
||||||
|
!store.pool_meta_writes_ready().await,
|
||||||
|
"releasing an unrelated writer must not clear the latch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pool_meta_save_gate_publishes_every_writer_release() {
|
||||||
|
let gate = PoolMetaSaveGate::default();
|
||||||
|
{
|
||||||
|
let mut state = gate.try_write().expect("an idle gate must admit a writer");
|
||||||
|
state.block_writes_after_fence_loss();
|
||||||
|
gate.published()
|
||||||
|
.ensure_write_safe("writer still active")
|
||||||
|
.expect("a writer's latch is not published before it releases the gate");
|
||||||
|
}
|
||||||
|
let err = gate
|
||||||
|
.published()
|
||||||
|
.ensure_write_safe("try_write released")
|
||||||
|
.expect_err("releasing a try_write guard must publish its latch");
|
||||||
|
let context = err
|
||||||
|
.pool_metadata_failure()
|
||||||
|
.expect("the published failure must carry pool metadata context");
|
||||||
|
assert_eq!(
|
||||||
|
(context.kind, context.phase),
|
||||||
|
(crate::error::PoolMetadataFailure::FenceLost, "format_heal"),
|
||||||
|
"the published copy must carry the writer's block context, not the external-fence fallback"
|
||||||
|
);
|
||||||
|
|
||||||
|
*gate.write().await = PoolMetaWriteState::default();
|
||||||
|
gate.published()
|
||||||
|
.ensure_write_safe("replacement released")
|
||||||
|
.expect("releasing a whole-state replacement must publish it");
|
||||||
|
|
||||||
|
// The published copy shares the replacement's live transaction latches.
|
||||||
|
let arm = {
|
||||||
|
let state = gate.write().await;
|
||||||
|
let mut arm = state.arm_transaction();
|
||||||
|
arm.phase = Some("publication");
|
||||||
|
arm
|
||||||
|
};
|
||||||
|
gate.published()
|
||||||
|
.ensure_write_safe("outcome armed")
|
||||||
|
.expect("an armed outcome alone must not block writes");
|
||||||
|
drop(arm);
|
||||||
|
let (block, aborted) = gate.published().write_block();
|
||||||
|
assert!(aborted, "an outcome latch must reach the published copy without another writer");
|
||||||
|
assert_eq!(
|
||||||
|
block.expect("the outcome latch must block writes").kind,
|
||||||
|
crate::error::PoolMetadataFailure::TransactionUnknown
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -27729,7 +28038,7 @@ mod tests {
|
|||||||
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||||
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
|
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
|
||||||
{
|
{
|
||||||
let state = store.pool_meta_save_gate.lock().await;
|
let state = store.pool_meta_save_gate.write().await;
|
||||||
let mut arm = state.arm_transaction();
|
let mut arm = state.arm_transaction();
|
||||||
arm.phase = Some("publication");
|
arm.phase = Some("publication");
|
||||||
drop(arm);
|
drop(arm);
|
||||||
@@ -27739,7 +28048,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
let original = store.pool_meta_write_gate_status().await;
|
let original = store.pool_meta_write_gate_status().await;
|
||||||
assert!(!original.writes_ready);
|
assert!(!original.writes_ready);
|
||||||
assert!(!original.check_timed_out);
|
|
||||||
assert!(!original.write_blocked);
|
assert!(!original.write_blocked);
|
||||||
assert!(original.transaction_aborted);
|
assert!(original.transaction_aborted);
|
||||||
assert_eq!(original.reason, Some("transaction_unknown"));
|
assert_eq!(original.reason, Some("transaction_unknown"));
|
||||||
@@ -27753,7 +28061,7 @@ mod tests {
|
|||||||
});
|
});
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
loop {
|
loop {
|
||||||
if store.pool_meta_save_gate.try_lock().is_err() {
|
if store.pool_meta_save_gate.try_write().is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
@@ -27771,7 +28079,7 @@ mod tests {
|
|||||||
let store = store.clone();
|
let store = store.clone();
|
||||||
async move { store.recover_pool_meta_transaction().await }
|
async move { store.recover_pool_meta_transaction().await }
|
||||||
});
|
});
|
||||||
store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
|
store.pool_meta_save_gate.write().await.block_writes_after_fence_loss();
|
||||||
drop(start_guard);
|
drop(start_guard);
|
||||||
assert!(!recovery.await.unwrap().unwrap());
|
assert!(!recovery.await.unwrap().unwrap());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -27819,7 +28127,7 @@ mod tests {
|
|||||||
*disks = vec![None; disk_count];
|
*disks = vec![None; disk_count];
|
||||||
}
|
}
|
||||||
|
|
||||||
let write_state = store.pool_meta_save_gate.lock().await;
|
let write_state = store.pool_meta_save_gate.write().await;
|
||||||
store
|
store
|
||||||
.acquire_pool_meta_read_guard(&write_state, "capacity probe")
|
.acquire_pool_meta_read_guard(&write_state, "capacity probe")
|
||||||
.await
|
.await
|
||||||
@@ -31043,7 +31351,7 @@ mod pools_tests {
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(cancelers),
|
decommission_cancelers: tokio::sync::RwLock::new(cancelers),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(super::PoolMetaWriteState::for_test_bootstrap()),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
})
|
})
|
||||||
@@ -33086,7 +33394,7 @@ mod pools_tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut save_guard = store.pool_meta_save_gate.lock().await;
|
let mut save_guard = store.pool_meta_save_gate.write().await;
|
||||||
snapshot
|
snapshot
|
||||||
.save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
|
.save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
|
||||||
.await
|
.await
|
||||||
@@ -33140,7 +33448,7 @@ mod pools_tests {
|
|||||||
let save_committed = committed.clone();
|
let save_committed = committed.clone();
|
||||||
let save_pending = pending.clone();
|
let save_pending = pending.clone();
|
||||||
let save_task = tokio::spawn(async move {
|
let save_task = tokio::spawn(async move {
|
||||||
let mut save_guard = save_store.pool_meta_save_gate.lock().await;
|
let mut save_guard = save_store.pool_meta_save_gate.write().await;
|
||||||
snapshot
|
snapshot
|
||||||
.save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
|
.save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
|
||||||
.await
|
.await
|
||||||
@@ -33157,7 +33465,7 @@ mod pools_tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
{
|
{
|
||||||
let save_guard = store.pool_meta_save_gate.lock().await;
|
let save_guard = store.pool_meta_save_gate.write().await;
|
||||||
save_guard
|
save_guard
|
||||||
.ensure_write_safe("cancelled pool metadata save")
|
.ensure_write_safe("cancelled pool metadata save")
|
||||||
.expect_err("a cancelled replica update must latch the sticky safety gate");
|
.expect_err("a cancelled replica update must latch the sticky safety gate");
|
||||||
@@ -33202,7 +33510,7 @@ mod pools_tests {
|
|||||||
let task_publish_started = publish_started.clone();
|
let task_publish_started = publish_started.clone();
|
||||||
let task_publish_release = publish_release.clone();
|
let task_publish_release = publish_release.clone();
|
||||||
let save_task = tokio::spawn(async move {
|
let save_task = tokio::spawn(async move {
|
||||||
let mut save_guard = save_store.pool_meta_save_gate.lock().await;
|
let mut save_guard = save_store.pool_meta_save_gate.write().await;
|
||||||
let outcome = snapshot
|
let outcome = snapshot
|
||||||
.save_no_lock_armed(vec![save_committed], &mut save_guard, PoolMetaPersistenceFence::OfflineRecovery, &[0])
|
.save_no_lock_armed(vec![save_committed], &mut save_guard, PoolMetaPersistenceFence::OfflineRecovery, &[0])
|
||||||
.await?;
|
.await?;
|
||||||
@@ -33225,7 +33533,7 @@ mod pools_tests {
|
|||||||
"the task should be aborted while publication is pending"
|
"the task should be aborted while publication is pending"
|
||||||
);
|
);
|
||||||
|
|
||||||
let save_guard = store.pool_meta_save_gate.lock().await;
|
let save_guard = store.pool_meta_save_gate.write().await;
|
||||||
save_guard
|
save_guard
|
||||||
.ensure_write_safe("cancelled pool metadata publication")
|
.ensure_write_safe("cancelled pool metadata publication")
|
||||||
.expect_err("cancellation after replica save but before publication must keep writes blocked");
|
.expect_err("cancellation after replica save but before publication must keep writes blocked");
|
||||||
@@ -35014,7 +35322,7 @@ mod pools_tests {
|
|||||||
let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
|
let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -35059,7 +35367,7 @@ mod pools_tests {
|
|||||||
let store = decommission_worker_test_store(pool_meta, vec![None]);
|
let store = decommission_worker_test_store(pool_meta, vec![None]);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -35130,7 +35438,7 @@ mod pools_tests {
|
|||||||
drop(pool_meta);
|
drop(pool_meta);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.ensure_write_safe("V1 unresolved-entry preflight")
|
.ensure_write_safe("V1 unresolved-entry preflight")
|
||||||
.expect("a deterministic capability rejection must not latch recovery");
|
.expect("a deterministic capability rejection must not latch recovery");
|
||||||
@@ -41598,7 +41906,7 @@ mod pools_tests {
|
|||||||
let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
|
let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -41762,7 +42070,8 @@ mod pools_tests {
|
|||||||
let (_directories, store, owner) = pending_convergence_fixture().await;
|
let (_directories, store, owner) = pending_convergence_fixture().await;
|
||||||
let interrupted_store = store.clone();
|
let interrupted_store = store.clone();
|
||||||
let interrupted_owner = owner.clone();
|
let interrupted_owner = owner.clone();
|
||||||
let err = store
|
let err =
|
||||||
|
store
|
||||||
.promote_queued_decommission_with_reload(0, &owner, || async move {
|
.promote_queued_decommission_with_reload(0, &owner, || async move {
|
||||||
match interrupt {
|
match interrupt {
|
||||||
"cancel" => interrupted_owner.cancel(),
|
"cancel" => interrupted_owner.cancel(),
|
||||||
@@ -41770,14 +42079,12 @@ mod pools_tests {
|
|||||||
interrupted_store.decommission_cancel(0).await?;
|
interrupted_store.decommission_cancel(0).await?;
|
||||||
interrupted_store.clear_decommission(0).await?;
|
interrupted_store.clear_decommission(0).await?;
|
||||||
}
|
}
|
||||||
_ => interrupted_store
|
_ => interrupted_store.pool_meta_save_gate.write().await.observe_replicas(
|
||||||
.pool_meta_save_gate
|
super::PoolMetaReplicaState {
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
repair_write_safe: false,
|
repair_write_safe: false,
|
||||||
}),
|
},
|
||||||
|
),
|
||||||
}
|
}
|
||||||
Err(Error::SlowDown)
|
Err(Error::SlowDown)
|
||||||
})
|
})
|
||||||
@@ -41932,7 +42239,7 @@ mod pools_tests {
|
|||||||
|
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -42072,7 +42379,7 @@ mod pools_tests {
|
|||||||
let store = decommission_worker_test_store(pool_meta, vec![None]);
|
let store = decommission_worker_test_store(pool_meta, vec![None]);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -42191,7 +42498,7 @@ mod pools_tests {
|
|||||||
let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
|
let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
|
||||||
store
|
store
|
||||||
.pool_meta_save_gate
|
.pool_meta_save_gate
|
||||||
.lock()
|
.write()
|
||||||
.await
|
.await
|
||||||
.observe_replicas(super::PoolMetaReplicaState {
|
.observe_replicas(super::PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
@@ -42459,7 +42766,7 @@ mod pools_tests {
|
|||||||
assert!(save_entered.load(Ordering::SeqCst));
|
assert!(save_entered.load(Ordering::SeqCst));
|
||||||
assert!(!cancel.is_finished(), "cancel must wait for the injected save");
|
assert!(!cancel.is_finished(), "cancel must wait for the injected save");
|
||||||
assert!(
|
assert!(
|
||||||
store.pool_meta_save_gate.try_lock().is_err(),
|
store.pool_meta_save_gate.try_write().is_err(),
|
||||||
"the cancel save must exclude stale full-document saves until publication"
|
"the cancel save must exclude stale full-document saves until publication"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -527,8 +527,8 @@ mod decommission_lock_order_tests {
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()),
|
rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]),
|
decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::new(
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(
|
||||||
other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(),
|
other_store.pool_meta_save_gate.write().await.independent_clone_for_test(),
|
||||||
),
|
),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
|
|||||||
@@ -3436,7 +3436,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: TokioMutex::new(()),
|
start_gate: TokioMutex::new(()),
|
||||||
pool_meta_save_gate: TokioMutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -977,6 +977,7 @@ async fn restore_delete_rollback_with_namespace_owner(
|
|||||||
.insert(backup_path.clone());
|
.insert(backup_path.clone());
|
||||||
os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await
|
os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await
|
||||||
};
|
};
|
||||||
|
let restored_backup = restored.is_ok();
|
||||||
match restored {
|
match restored {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(DiskError::FileNotFound) if remove_new_metadata => {
|
Err(DiskError::FileNotFound) if remove_new_metadata => {
|
||||||
@@ -1006,6 +1007,16 @@ async fn restore_delete_rollback_with_namespace_owner(
|
|||||||
os::fsync_dir_with_owner(object_dir, namespace_owner.clone())
|
os::fsync_dir_with_owner(object_dir, namespace_owner.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(to_file_error)?;
|
.map_err(to_file_error)?;
|
||||||
|
}
|
||||||
|
if restored_backup {
|
||||||
|
// Renaming two hardlinks to the same inode can leave the backup in place.
|
||||||
|
match os::remove_file_with_owner(&backup_path, namespace_owner.clone()).await {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||||
|
Err(err) => return Err(to_file_error(err).into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if effective_durability(volume).syncs_commit_metadata() {
|
||||||
os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone())
|
os::fsync_dir_with_owner(&rollback_path, namespace_owner.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(to_file_error)?;
|
.map_err(to_file_error)?;
|
||||||
@@ -3794,6 +3805,25 @@ async fn read_dir_entries_with_walk_stall(path: &Path, count: i32, stall: Option
|
|||||||
const WALK_DIR_BATCH_SIZE: usize = 32;
|
const WALK_DIR_BATCH_SIZE: usize = 32;
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
const WALK_DIR_BATCH_SIZE: usize = 256;
|
const WALK_DIR_BATCH_SIZE: usize = 256;
|
||||||
|
|
||||||
|
async fn read_identifier_tree_entries(path: &Path, leaf: &str, stall: Option<Duration>) -> Result<Vec<String>> {
|
||||||
|
let mut entries = read_dir_with_walk_stall(path.to_path_buf(), stall).await?;
|
||||||
|
let mut names = Vec::new();
|
||||||
|
loop {
|
||||||
|
let (next, batch, eof) = read_dir_entry_batch_with_walk_stall(entries, READ_DIR_BATCH_SIZE, stall).await?;
|
||||||
|
entries = next;
|
||||||
|
// Filter before retaining or sorting the inventory. A report flood
|
||||||
|
// costs directory iteration, not one retained name/metadata read per report.
|
||||||
|
names.extend(batch.into_iter().filter(|name| {
|
||||||
|
name.strip_suffix(SLASH_SEPARATOR)
|
||||||
|
.is_some_and(|component| component == leaf || super::is_identifier_tree_component(component))
|
||||||
|
}));
|
||||||
|
if eof {
|
||||||
|
return Ok(names);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const READ_DIR_BATCH_SIZE: usize = 512;
|
const READ_DIR_BATCH_SIZE: usize = 512;
|
||||||
|
|
||||||
async fn read_dir_with_walk_stall(path: PathBuf, stall: Option<Duration>) -> Result<std::fs::ReadDir> {
|
async fn read_dir_with_walk_stall(path: PathBuf, stall: Option<Duration>) -> Result<std::fs::ReadDir> {
|
||||||
@@ -11746,6 +11776,14 @@ impl LocalDisk {
|
|||||||
W: AsyncWrite + Unpin + Send + 'a,
|
W: AsyncWrite + Unpin + Send + 'a,
|
||||||
{
|
{
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
|
if opts.identifier_tree_leaf.is_some() {
|
||||||
|
let relative = current
|
||||||
|
.strip_prefix(&opts.base_dir)
|
||||||
|
.ok_or_else(|| DiskError::other("identifier-tree walk escaped its root"))?;
|
||||||
|
if relative.split('/').filter(|component| !component.is_empty()).count() > 512 {
|
||||||
|
return Err(DiskError::other("identifier-tree walk exceeds its directory depth limit"));
|
||||||
|
}
|
||||||
|
}
|
||||||
// The part of forward_to below this directory, taken before `current`
|
// The part of forward_to below this directory, taken before `current`
|
||||||
// loses its trailing slash.
|
// loses its trailing slash.
|
||||||
let forward_rest = opts
|
let forward_rest = opts
|
||||||
@@ -11778,7 +11816,12 @@ impl LocalDisk {
|
|||||||
// as a single logical read in API semantics.
|
// as a single logical read in API semantics.
|
||||||
let read_dir_started = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
let read_dir_started = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||||
let dir_path_abs = self.io_get_object_path(&opts.bucket, current.trim_start_matches(SLASH_SEPARATOR))?;
|
let dir_path_abs = self.io_get_object_path(&opts.bucket, current.trim_start_matches(SLASH_SEPARATOR))?;
|
||||||
let read_dir_result = match read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await {
|
let read_dir_result = if let Some(leaf) = opts.identifier_tree_leaf.as_deref() {
|
||||||
|
read_identifier_tree_entries(&dir_path_abs, leaf, stall).await
|
||||||
|
} else {
|
||||||
|
read_dir_entries_with_walk_stall(&dir_path_abs, -1, stall).await
|
||||||
|
};
|
||||||
|
let read_dir_result = match read_dir_result {
|
||||||
Err(err) if err == Error::FileNotFound && !skip_access_checks(&opts.bucket) => {
|
Err(err) if err == Error::FileNotFound && !skip_access_checks(&opts.bucket) => {
|
||||||
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
|
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
|
||||||
if let Err(access_err) = cached_access(&volume_dir).await {
|
if let Err(access_err) = cached_access(&volume_dir).await {
|
||||||
@@ -12075,6 +12118,16 @@ impl LocalDisk {
|
|||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(leaf) = opts.identifier_tree_leaf.as_deref()
|
||||||
|
&& entry != leaf
|
||||||
|
{
|
||||||
|
// Identifier directories can be namespaces at any depth,
|
||||||
|
// including components named tables or maintenance. Their
|
||||||
|
// own object metadata is outside this selected leaf scan.
|
||||||
|
schedule_dir(&mut dir_stack, format!("{name}/"), true, None, true, false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let mut meta = MetaCacheEntry {
|
let mut meta = MetaCacheEntry {
|
||||||
name,
|
name,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -12124,7 +12177,8 @@ impl LocalDisk {
|
|||||||
} else {
|
} else {
|
||||||
adaptive_batch_cap
|
adaptive_batch_cap
|
||||||
};
|
};
|
||||||
let batch_names = if batch_limit > 1
|
let batch_names = if opts.identifier_tree_leaf.is_none()
|
||||||
|
&& batch_limit > 1
|
||||||
&& dir_objes.is_empty()
|
&& dir_objes.is_empty()
|
||||||
&& dir_stack.iter().all(|(_, _, _, scan_required, _)| !*scan_required)
|
&& dir_stack.iter().all(|(_, _, _, scan_required, _)| !*scan_required)
|
||||||
{
|
{
|
||||||
@@ -12177,6 +12231,11 @@ impl LocalDisk {
|
|||||||
write_metacache_obj(out, &meta).await?;
|
write_metacache_obj(out, &meta).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.identifier_tree_leaf.is_some() {
|
||||||
|
// A dotted leaf cannot contain identifier-tree descendants.
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let has_namespace_child_candidate = read.has_namespace_child_candidate;
|
let has_namespace_child_candidate = read.has_namespace_child_candidate;
|
||||||
let dir_to_skip: HashSet<String> = if has_namespace_child_candidate || !dir_objes.is_empty() {
|
let dir_to_skip: HashSet<String> = if has_namespace_child_candidate || !dir_objes.is_empty() {
|
||||||
read.data_dirs.into_iter().map(|dir| dir.to_string()).collect()
|
read.data_dirs.into_iter().map(|dir| dir.to_string()).collect()
|
||||||
@@ -12223,6 +12282,9 @@ impl LocalDisk {
|
|||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if err == Error::FileNotFound || err == Error::IsNotRegular {
|
if err == Error::FileNotFound || err == Error::IsNotRegular {
|
||||||
|
if opts.identifier_tree_leaf.is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
adaptive_batch_cap = 1;
|
adaptive_batch_cap = 1;
|
||||||
// NOT an object, append to stack (with slash)
|
// NOT an object, append to stack (with slash)
|
||||||
// If dirObject, but no metadata (which is unexpected) we skip it.
|
// If dirObject, but no metadata (which is unexpected) we skip it.
|
||||||
@@ -15431,6 +15493,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
super::lifecycle_test_io::observe(&opts.bucket, None);
|
super::lifecycle_test_io::observe(&opts.bucket, None);
|
||||||
self.wait_for_startup_cleanup().await;
|
self.wait_for_startup_cleanup().await;
|
||||||
|
|
||||||
|
if let Some(leaf) = opts.identifier_tree_leaf.as_deref()
|
||||||
|
&& (!opts.recursive || !opts.base_dir.ends_with('/') || !super::is_identifier_tree_leaf(leaf))
|
||||||
|
{
|
||||||
|
return Err(DiskError::other("invalid identifier-tree walk options"));
|
||||||
|
}
|
||||||
|
|
||||||
// Callers that do not pin a stall budget inherit the configured default, so
|
// Callers that do not pin a stall budget inherit the configured default, so
|
||||||
// every local walk keeps a liveness bound even when the wrapper-level total
|
// every local walk keeps a liveness bound even when the wrapper-level total
|
||||||
// timeout is skipped.
|
// timeout is skipped.
|
||||||
@@ -15457,7 +15525,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
|
|
||||||
let mut skip_current_dir_object = false;
|
let mut skip_current_dir_object = false;
|
||||||
let mut multipart_dir_to_skip: HashSet<String> = HashSet::new();
|
let mut multipart_dir_to_skip: HashSet<String> = HashSet::new();
|
||||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
if opts.identifier_tree_leaf.is_none() && opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||||
if let Ok(data) = with_walk_stall_timeout(
|
if let Ok(data) = with_walk_stall_timeout(
|
||||||
stall,
|
stall,
|
||||||
self.read_listing_metadata(
|
self.read_listing_metadata(
|
||||||
@@ -19407,6 +19475,55 @@ mod test {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn identifier_tree_walk_filters_report_batches_and_bounds_depth() {
|
||||||
|
let dir = tempfile::tempdir().expect("create disk fixture");
|
||||||
|
let root = dir.path().join("catalog/prefix");
|
||||||
|
fs::create_dir_all(root.join("sales"))
|
||||||
|
.await
|
||||||
|
.expect("create identifier directory");
|
||||||
|
fs::create_dir_all(root.join("table-entry.json"))
|
||||||
|
.await
|
||||||
|
.expect("create selected leaf");
|
||||||
|
for index in 0..READ_DIR_BATCH_SIZE * 3 + 1 {
|
||||||
|
fs::create_dir(root.join(format!("report-{index}.json")))
|
||||||
|
.await
|
||||||
|
.expect("create report directory");
|
||||||
|
}
|
||||||
|
let mut names = read_identifier_tree_entries(&root, "table-entry.json", Some(Duration::from_secs(5)))
|
||||||
|
.await
|
||||||
|
.expect("all directory batches should be filtered");
|
||||||
|
names.sort();
|
||||||
|
assert_eq!(names, ["sales/", "table-entry.json/"]);
|
||||||
|
|
||||||
|
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("valid endpoint");
|
||||||
|
let disk = LocalDisk::new(&endpoint, false).await.expect("create local disk");
|
||||||
|
let options = WalkDirOptions {
|
||||||
|
bucket: "catalog".to_string(),
|
||||||
|
base_dir: "prefix/".to_string(),
|
||||||
|
recursive: true,
|
||||||
|
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut output = Vec::new();
|
||||||
|
let mut writer = MetacacheWriter::new(&mut output);
|
||||||
|
let mut count = 0;
|
||||||
|
let error = disk
|
||||||
|
.scan_dir(
|
||||||
|
format!("prefix/{}", "a/".repeat(513)),
|
||||||
|
String::new(),
|
||||||
|
&options,
|
||||||
|
&mut writer,
|
||||||
|
&mut count,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("excessive depth must fail before filesystem I/O");
|
||||||
|
assert!(error.to_string().contains("directory depth limit"));
|
||||||
|
assert_eq!(count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
// #4644: the stall budget bounds a single drive read, never the walk as a
|
// #4644: the stall budget bounds a single drive read, never the walk as a
|
||||||
// whole. A walk that keeps making progress must survive even when the total
|
// whole. A walk that keeps making progress must survive even when the total
|
||||||
// time spent blocked on a slow consumer dwarfs the stall timeout.
|
// time spent blocked on a slow consumer dwarfs the stall timeout.
|
||||||
|
|||||||
@@ -7240,6 +7240,9 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn delete_recovery_fused_prepare_checkpoint_reopens_before_head_publication() {
|
async fn delete_recovery_fused_prepare_checkpoint_reopens_before_head_publication() {
|
||||||
|
use std::os::unix::fs::MetadataExt;
|
||||||
|
|
||||||
|
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||||
let (_temp, disk, mut first, _) = fixture().await;
|
let (_temp, disk, mut first, _) = fixture().await;
|
||||||
first.version_id = Some(Uuid::new_v4());
|
first.version_id = Some(Uuid::new_v4());
|
||||||
let mut second = first.clone();
|
let mut second = first.clone();
|
||||||
@@ -7263,12 +7266,25 @@ mod tests {
|
|||||||
let source = source_root.join(transaction.to_string());
|
let source = source_root.join(transaction.to_string());
|
||||||
let destination = copy_root.join(transaction.to_string());
|
let destination = copy_root.join(transaction.to_string());
|
||||||
std::fs::create_dir(&destination).expect("create checkpoint transaction");
|
std::fs::create_dir(&destination).expect("create checkpoint transaction");
|
||||||
std::fs::copy(source_root.join(STORAGE_FORMAT_FILE), copy_root.join(STORAGE_FORMAT_FILE))
|
let source_head = source_root.join(STORAGE_FORMAT_FILE);
|
||||||
.expect("capture head before publication");
|
let source_head_meta = std::fs::metadata(&source_head).expect("stat source head");
|
||||||
|
let copy_head = copy_root.join(STORAGE_FORMAT_FILE);
|
||||||
|
std::fs::copy(&source_head, ©_head).expect("capture head before publication");
|
||||||
for entry in std::fs::read_dir(source).expect("read published preparation") {
|
for entry in std::fs::read_dir(source).expect("read published preparation") {
|
||||||
let entry = entry.expect("prepared file");
|
let entry = entry.expect("prepared file");
|
||||||
assert!(entry.file_type().expect("prepared file type").is_file());
|
assert!(entry.file_type().expect("prepared file type").is_file());
|
||||||
std::fs::copy(entry.path(), destination.join(entry.file_name())).expect("capture prepared file");
|
let dst = destination.join(entry.file_name());
|
||||||
|
if entry.file_name() == STORAGE_FORMAT_FILE_BACKUP {
|
||||||
|
let backup_meta = entry.metadata().expect("stat prepared backup");
|
||||||
|
assert_eq!(
|
||||||
|
(backup_meta.dev(), backup_meta.ino()),
|
||||||
|
(source_head_meta.dev(), source_head_meta.ino()),
|
||||||
|
"fused preparation must hardlink backup to live head"
|
||||||
|
);
|
||||||
|
std::fs::hard_link(©_head, &dst).expect("preserve hardlinked backup");
|
||||||
|
} else {
|
||||||
|
std::fs::copy(entry.path(), dst).expect("capture prepared file");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let opts = DeleteOptions {
|
let opts = DeleteOptions {
|
||||||
@@ -7292,10 +7308,42 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("actual preparation checkpoint must remain recoverable");
|
.expect("actual preparation checkpoint must remain recoverable");
|
||||||
assert_eq!(inspection.state, DeleteRecoveryState::NewPrepared);
|
assert_eq!(inspection.state, DeleteRecoveryState::NewPrepared);
|
||||||
|
let reopened_tx = reopened
|
||||||
|
.io_get_object_path("recovery", &format!("object/{transaction}"))
|
||||||
|
.expect("reopened transaction path");
|
||||||
|
os::fsync_dir_recorder::set_failure(&reopened_tx, ErrorKind::Other);
|
||||||
|
assert!(
|
||||||
reopened
|
reopened
|
||||||
.settle_delete_transaction("recovery", "object", transaction, inspection.fingerprint, DeleteRecoveryAction::Rollback)
|
.settle_delete_transaction(
|
||||||
|
"recovery",
|
||||||
|
"object",
|
||||||
|
transaction,
|
||||||
|
inspection.fingerprint,
|
||||||
|
DeleteRecoveryAction::Rollback
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"failed transaction directory sync after backup removal must retain journal"
|
||||||
|
);
|
||||||
|
assert!(reopened_tx.join(PREPARED).is_file());
|
||||||
|
assert!(!reopened_tx.join(STORAGE_FORMAT_FILE_BACKUP).exists());
|
||||||
|
let retry_inspection = reopened
|
||||||
|
.delete_recovery_inspect("recovery", "object", transaction)
|
||||||
|
.await
|
||||||
|
.expect("inspect interrupted rollback");
|
||||||
|
assert_eq!(retry_inspection.state, DeleteRecoveryState::NewPrepared);
|
||||||
|
let settled = reopened
|
||||||
|
.settle_delete_transaction(
|
||||||
|
"recovery",
|
||||||
|
"object",
|
||||||
|
transaction,
|
||||||
|
retry_inspection.fingerprint,
|
||||||
|
DeleteRecoveryAction::Rollback,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
.expect("roll back interrupted preparation");
|
.expect("roll back interrupted preparation");
|
||||||
|
assert_eq!(settled.state, DeleteRecoveryState::Absent, "same-inode backup must not survive rollback");
|
||||||
|
assert!(!checkpoint_root.join(transaction.to_string()).exists());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fs::read(checkpoint_root.join(STORAGE_FORMAT_FILE))
|
fs::read(checkpoint_root.join(STORAGE_FORMAT_FILE))
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -2222,6 +2222,36 @@ pub struct WalkDirOptions {
|
|||||||
// Override the remote stream stall timeout for long background walks.
|
// Override the remote stream stall timeout for long background walks.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub stall_timeout_ms: Option<u64>,
|
pub stall_timeout_ms: Option<u64>,
|
||||||
|
|
||||||
|
/// Select this leaf below identifier-like directory components. The root is
|
||||||
|
/// `base_dir`; unrelated object metadata must not be read or returned.
|
||||||
|
/// Older peers ignore this additive field, so consumers must also filter.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub identifier_tree_leaf: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_identifier_tree_component(value: &str) -> bool {
|
||||||
|
!value.is_empty()
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_identifier_tree_leaf(value: &str) -> bool {
|
||||||
|
value.len() <= 255
|
||||||
|
&& value.contains('.')
|
||||||
|
&& !matches!(value, "." | "..")
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn identifier_tree_leaf_matches(prefix: &str, leaf: &str, object: &str) -> bool {
|
||||||
|
let Some(relative) = object.strip_prefix(prefix) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let mut components = relative.split('/');
|
||||||
|
components.next_back() == Some(leaf) && components.all(is_identifier_tree_component)
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WalkDirOptions {
|
impl WalkDirOptions {
|
||||||
@@ -3157,6 +3187,7 @@ mod tests {
|
|||||||
skip_total_timeout: false,
|
skip_total_timeout: false,
|
||||||
timeout_ms: Some(10_000),
|
timeout_ms: Some(10_000),
|
||||||
stall_timeout_ms: Some(20_000),
|
stall_timeout_ms: Some(20_000),
|
||||||
|
identifier_tree_leaf: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
assert_eq!(opts.bucket, "test-bucket");
|
assert_eq!(opts.bucket, "test-bucket");
|
||||||
@@ -3191,6 +3222,69 @@ mod tests {
|
|||||||
assert!(!decoded.skip_hidden_prefix_check);
|
assert!(!decoded.skip_hidden_prefix_check);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn identifier_tree_leaf_selection_preserves_nested_identifier_directories() {
|
||||||
|
let prefix = "catalog/namespaces/";
|
||||||
|
let leaf = "table-entry.json";
|
||||||
|
for relative in [
|
||||||
|
"sales/tables/orders/table-entry.json",
|
||||||
|
"sales/tables/orders/maintenance/tables/owner/table-entry.json",
|
||||||
|
"maintenance/tables/tables/table-entry.json",
|
||||||
|
] {
|
||||||
|
assert!(identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
|
||||||
|
}
|
||||||
|
for relative in [
|
||||||
|
"sales/tables/orders/not-table-entry.json",
|
||||||
|
"sales/tables/orders/table-entry.json/child",
|
||||||
|
"sales/report.json/tables/owner/table-entry.json",
|
||||||
|
"sales//tables/owner/table-entry.json",
|
||||||
|
"sales/../tables/owner/table-entry.json",
|
||||||
|
] {
|
||||||
|
assert!(!identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
|
||||||
|
}
|
||||||
|
assert!(!identifier_tree_leaf_matches(prefix, leaf, "other/sales/tables/orders/table-entry.json"));
|
||||||
|
assert!(is_identifier_tree_leaf(leaf));
|
||||||
|
for invalid in ["", ".", "..", "table-entry", "table/entry.json", "table\\entry.json"] {
|
||||||
|
assert!(!is_identifier_tree_leaf(invalid), "invalid selector {invalid}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn identifier_tree_walk_options_are_additive_on_json_and_messagepack_maps() {
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct LegacyView {
|
||||||
|
bucket: String,
|
||||||
|
base_dir: String,
|
||||||
|
recursive: bool,
|
||||||
|
}
|
||||||
|
let options = WalkDirOptions {
|
||||||
|
bucket: "catalog".to_string(),
|
||||||
|
base_dir: "namespaces/".to_string(),
|
||||||
|
recursive: true,
|
||||||
|
identifier_tree_leaf: Some("table-entry.json".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let json = serde_json::to_vec(&options).expect("encode current HTTP options");
|
||||||
|
let messagepack = rmp_serde::to_vec_named(&options).expect("encode legacy RPC map");
|
||||||
|
for old in [
|
||||||
|
serde_json::from_slice::<LegacyView>(&json).expect("old JSON readers ignore the additive field"),
|
||||||
|
rmp_serde::from_slice::<LegacyView>(&messagepack).expect("old MessagePack readers ignore the additive field"),
|
||||||
|
] {
|
||||||
|
assert_eq!(old.bucket, options.bucket);
|
||||||
|
assert_eq!(old.base_dir, options.base_dir);
|
||||||
|
assert!(old.recursive);
|
||||||
|
}
|
||||||
|
let mut old = serde_json::to_value(&options).expect("encode legacy options");
|
||||||
|
old.as_object_mut().expect("options are a map").remove("identifier_tree_leaf");
|
||||||
|
let from_json: WalkDirOptions = serde_json::from_value(old.clone()).expect("decode old HTTP options");
|
||||||
|
let from_messagepack: WalkDirOptions =
|
||||||
|
rmp_serde::from_slice(&rmp_serde::to_vec_named(&old).expect("encode old RPC map")).expect("decode old RPC map");
|
||||||
|
assert!(from_json.identifier_tree_leaf.is_none());
|
||||||
|
assert!(from_messagepack.identifier_tree_leaf.is_none());
|
||||||
|
assert_eq!(from_json.base_dir, options.base_dir);
|
||||||
|
assert_eq!(from_messagepack.base_dir, options.base_dir);
|
||||||
|
}
|
||||||
|
|
||||||
/// Test DeleteOptions structure
|
/// Test DeleteOptions structure
|
||||||
#[test]
|
#[test]
|
||||||
fn test_delete_options() {
|
fn test_delete_options() {
|
||||||
|
|||||||
@@ -593,7 +593,7 @@ impl ECStore {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
|
crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
|
||||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||||
pool_meta_guard.ensure_write_safe(stage)?;
|
pool_meta_guard.ensure_write_safe(stage)?;
|
||||||
let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
|
let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
|
||||||
let pool_meta = self
|
let pool_meta = self
|
||||||
@@ -625,7 +625,7 @@ impl ECStore {
|
|||||||
{
|
{
|
||||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
||||||
// shared fences admit a peer without waiting for the run's own entries.
|
// shared fences admit a peer without waiting for the run's own entries.
|
||||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||||
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
||||||
let pool_meta = self
|
let pool_meta = self
|
||||||
@@ -654,7 +654,7 @@ impl ECStore {
|
|||||||
{
|
{
|
||||||
// The admission read fences must already be dropped. Reacquire in the
|
// The admission read fences must already be dropped. Reacquire in the
|
||||||
// original order and revalidate every durable boundary across that gap.
|
// original order and revalidate every durable boundary across that gap.
|
||||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
|
||||||
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
||||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||||
let pool_meta = self
|
let pool_meta = self
|
||||||
@@ -1714,7 +1714,7 @@ mod tests {
|
|||||||
persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
|
persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
|
||||||
.await
|
.await
|
||||||
.expect("initialized pool metadata identity should persist");
|
.expect("initialized pool metadata identity should persist");
|
||||||
*store.pool_meta_save_gate.lock().await = write_state;
|
*store.pool_meta_save_gate.write().await = write_state;
|
||||||
// Simulate physical loss; raw deletion of authoritative pool.bin is
|
// Simulate physical loss; raw deletion of authoritative pool.bin is
|
||||||
// deliberately rejected by the native metadata mutation guard.
|
// deliberately rejected by the native metadata mutation guard.
|
||||||
for pool in &store.pools {
|
for pool in &store.pools {
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
|
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
});
|
});
|
||||||
@@ -335,7 +335,7 @@ async fn test_pool_stores_with_contexts(
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
|
decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(pool_meta_write_state.independent_clone_for_test()),
|
||||||
ctx: store_ctx,
|
ctx: store_ctx,
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3412,7 +3412,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -93,6 +93,7 @@ use super::{
|
|||||||
delete_tier_mutation_intent_record_if_current, list_tier_coordinator_mutation_intent_records,
|
delete_tier_mutation_intent_record_if_current, list_tier_coordinator_mutation_intent_records,
|
||||||
list_tier_mutation_intent_records, load_tier_coordinator_mutation_intent_record_with_etag,
|
list_tier_mutation_intent_records, load_tier_coordinator_mutation_intent_record_with_etag,
|
||||||
load_tier_mutation_intent_record_with_etag, save_tier_coordinator_mutation_intent_record_if_absent,
|
load_tier_mutation_intent_record_with_etag, save_tier_coordinator_mutation_intent_record_if_absent,
|
||||||
|
save_tier_mutation_intent_record_if_current,
|
||||||
},
|
},
|
||||||
warm_backend::WarmBackendImpl,
|
warm_backend::WarmBackendImpl,
|
||||||
};
|
};
|
||||||
@@ -620,7 +621,11 @@ struct CommittedMutationRecoveryPlan {
|
|||||||
|
|
||||||
impl TierMutationRecoverySnapshot {
|
impl TierMutationRecoverySnapshot {
|
||||||
fn requires_config_lock(&self) -> bool {
|
fn requires_config_lock(&self) -> bool {
|
||||||
self.has_committed_mutation_blocks || self.intents.iter().any(|recovered| recovered.has_coordinator_record)
|
self.has_committed_mutation_blocks
|
||||||
|
|| self.intents.iter().any(|recovered| {
|
||||||
|
recovered.has_coordinator_record
|
||||||
|
|| (recovered.has_peer_record && recovered.intent.state == TierMutationIntentState::Prepared)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn allowance(&self) -> MutationBlockAllowance {
|
fn allowance(&self) -> MutationBlockAllowance {
|
||||||
@@ -5102,6 +5107,9 @@ impl TierConfigMgr {
|
|||||||
|
|
||||||
Self::reconcile_terminal_dual_prefix_mutation_intents(api.clone(), &mut snapshot).await?;
|
Self::reconcile_terminal_dual_prefix_mutation_intents(api.clone(), &mut snapshot).await?;
|
||||||
Self::recover_prepared_coordinator_mutation_intents(api.clone(), &mut snapshot.coordinator_intents).await?;
|
Self::recover_prepared_coordinator_mutation_intents(api.clone(), &mut snapshot.coordinator_intents).await?;
|
||||||
|
if config_lock.is_some() {
|
||||||
|
Self::recover_expired_peer_only_prepares(api.clone(), &snapshot).await?;
|
||||||
|
}
|
||||||
|
|
||||||
snapshot = Self::load_and_reconcile_mutation_recovery_snapshot(handle, api.clone(), true).await?;
|
snapshot = Self::load_and_reconcile_mutation_recovery_snapshot(handle, api.clone(), true).await?;
|
||||||
if snapshot.requires_config_lock() && config_lock.is_none() {
|
if snapshot.requires_config_lock() && config_lock.is_none() {
|
||||||
@@ -5502,6 +5510,58 @@ impl TierConfigMgr {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn recover_expired_peer_only_prepares<S>(api: Arc<S>, snapshot: &TierMutationRecoverySnapshot) -> io::Result<()>
|
||||||
|
where
|
||||||
|
S: EcstoreObjectIO,
|
||||||
|
{
|
||||||
|
let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
|
||||||
|
let orphaned = snapshot
|
||||||
|
.intents
|
||||||
|
.iter()
|
||||||
|
.filter(|recovered| {
|
||||||
|
recovered.has_peer_record
|
||||||
|
&& !recovered.has_coordinator_record
|
||||||
|
&& recovered.intent.state == TierMutationIntentState::Prepared
|
||||||
|
&& Self::tier_mutation_tombstone_retention_elapsed(&recovered.intent, now)
|
||||||
|
})
|
||||||
|
.map(|recovered| recovered.intent.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if orphaned.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// The caller holds the tier-config namespace lock. Only the exact
|
||||||
|
// persisted candidate proves a commit; an old or different config
|
||||||
|
// leaves the peer fence intact.
|
||||||
|
let loaded = load_tier_config_for_recovery(api.clone()).await?;
|
||||||
|
let Some(config_etag) = loaded.etag else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for intent in orphaned {
|
||||||
|
if loaded.persisted_digest != Some(intent.candidate_digest) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let _mutation_guard = acquire_tier_mutation_mutex(intent.mutation_id).await;
|
||||||
|
let (mut current, record_etag) =
|
||||||
|
match load_tier_mutation_intent_record_with_etag(api.clone(), intent.mutation_id).await {
|
||||||
|
Ok(record) => record,
|
||||||
|
Err(Error::ConfigNotFound) => continue,
|
||||||
|
Err(err) => return Err(tier_mutation_replay_error(err)),
|
||||||
|
};
|
||||||
|
if current != intent {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
current
|
||||||
|
.advance(TierMutationIntentState::Committed, Some(config_etag.clone()))
|
||||||
|
.map_err(io::Error::other)?;
|
||||||
|
match save_tier_mutation_intent_record_if_current(api.clone(), ¤t, &record_etag).await {
|
||||||
|
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||||
|
Err(err) => return Err(tier_mutation_replay_error(err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn recover_prepared_coordinator_mutation_intents<S>(api: Arc<S>, intents: &mut [TierMutationIntent]) -> io::Result<()>
|
async fn recover_prepared_coordinator_mutation_intents<S>(api: Arc<S>, intents: &mut [TierMutationIntent]) -> io::Result<()>
|
||||||
where
|
where
|
||||||
S: EcstoreObjectIO,
|
S: EcstoreObjectIO,
|
||||||
@@ -13379,6 +13439,208 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reload_recovers_expired_peer_only_prepare_for_persisted_add() {
|
||||||
|
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||||
|
|
||||||
|
let store = Arc::new(CasConfigStore::default());
|
||||||
|
let previous = empty_mgr();
|
||||||
|
let mut candidate = empty_mgr();
|
||||||
|
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||||
|
let targets = build_tier_mutation_affected_targets(
|
||||||
|
TierMutationIntentKind::Add,
|
||||||
|
HashSet::from(["COLD-A".to_string()]),
|
||||||
|
&previous,
|
||||||
|
&candidate,
|
||||||
|
)
|
||||||
|
.expect("added tier should have a target");
|
||||||
|
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||||
|
.expect("intent should build")
|
||||||
|
.expect("add should require an intent");
|
||||||
|
intent.expires_at_unix_nanos = 1;
|
||||||
|
candidate
|
||||||
|
.save_tiering_config_if_current(store.clone(), None)
|
||||||
|
.await
|
||||||
|
.expect("candidate should persist");
|
||||||
|
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||||
|
.await
|
||||||
|
.expect("peer prepare should persist without a coordinator record");
|
||||||
|
|
||||||
|
let manager = TierConfigMgr::new();
|
||||||
|
TIER_MUTATION_TEST_PEERS
|
||||||
|
.scope(Vec::new(), async {
|
||||||
|
TIER_DRIVER_TEST_FACTORY
|
||||||
|
.scope(healthy_driver_factory(), async {
|
||||||
|
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||||
|
.await
|
||||||
|
.expect("matching saved candidate should recover");
|
||||||
|
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||||
|
.await
|
||||||
|
.expect("recovered tier should be readable");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
TierConfigMgr::load_tier_mutation_intents(store)
|
||||||
|
.await
|
||||||
|
.expect("peer scan should succeed")
|
||||||
|
.iter()
|
||||||
|
.all(|record| record.state != TierMutationIntentState::Prepared),
|
||||||
|
"the orphaned prepare must no longer block later reloads"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reload_keeps_peer_only_prepare_without_commit_proof() {
|
||||||
|
use crate::services::tier::tier_mutation_intent::{
|
||||||
|
save_tier_mutation_intent_record, tier_mutation_intent_record_object_name,
|
||||||
|
};
|
||||||
|
|
||||||
|
for case in [
|
||||||
|
"unexpired",
|
||||||
|
"recently_expired",
|
||||||
|
"different_candidate",
|
||||||
|
"missing_config",
|
||||||
|
"concurrent_change",
|
||||||
|
] {
|
||||||
|
let store = Arc::new(CasConfigStore::default());
|
||||||
|
let previous = empty_mgr();
|
||||||
|
let mut candidate = empty_mgr();
|
||||||
|
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||||
|
let targets = build_tier_mutation_affected_targets(
|
||||||
|
TierMutationIntentKind::Add,
|
||||||
|
HashSet::from(["COLD-A".to_string()]),
|
||||||
|
&previous,
|
||||||
|
&candidate,
|
||||||
|
)
|
||||||
|
.expect("added tier should have a target");
|
||||||
|
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||||
|
.expect("intent should build")
|
||||||
|
.expect("add should require an intent");
|
||||||
|
intent.expires_at_unix_nanos = match case {
|
||||||
|
"unexpired" => tier_mutation_intent_expiry_unix_nanos(),
|
||||||
|
"recently_expired" => {
|
||||||
|
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).expect("test clock should fit")
|
||||||
|
- 1_000_000_000
|
||||||
|
}
|
||||||
|
_ => 1,
|
||||||
|
};
|
||||||
|
if case == "different_candidate" {
|
||||||
|
intent.candidate_digest = [0x51; 32];
|
||||||
|
}
|
||||||
|
if case != "missing_config" {
|
||||||
|
candidate
|
||||||
|
.save_tiering_config_if_current(store.clone(), None)
|
||||||
|
.await
|
||||||
|
.expect("candidate should persist");
|
||||||
|
}
|
||||||
|
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||||
|
.await
|
||||||
|
.expect("peer prepare should persist");
|
||||||
|
let expected = if case == "concurrent_change" {
|
||||||
|
let mut changed = intent.clone();
|
||||||
|
changed.candidate_digest = [0x52; 32];
|
||||||
|
let object = tier_mutation_intent_record_object_name(intent.mutation_id).expect("record path should build");
|
||||||
|
store
|
||||||
|
.rewrite_on_next_if_match(object, changed.encode().expect("changed intent should encode"))
|
||||||
|
.await;
|
||||||
|
changed
|
||||||
|
} else {
|
||||||
|
intent
|
||||||
|
};
|
||||||
|
|
||||||
|
let manager = TierConfigMgr::new();
|
||||||
|
TIER_MUTATION_TEST_PEERS
|
||||||
|
.scope(Vec::new(), async {
|
||||||
|
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||||
|
.await
|
||||||
|
.expect("unproven prepare should remain fenced");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let error = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||||
|
Ok(_) => panic!("{case}: unproven prepare must block reads"),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
assert!(error.message.contains("being replaced"), "{case}: {error}");
|
||||||
|
let records = TierConfigMgr::load_tier_mutation_intents(store)
|
||||||
|
.await
|
||||||
|
.expect("peer scan should succeed");
|
||||||
|
assert_eq!(records, vec![expected], "{case}: prepared evidence must remain unchanged");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn peer_only_prepare_recovery_retains_fence_until_peer_commit_succeeds() {
|
||||||
|
use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
|
||||||
|
|
||||||
|
let store = Arc::new(CasConfigStore::default());
|
||||||
|
let previous = empty_mgr();
|
||||||
|
let mut candidate = empty_mgr();
|
||||||
|
candidate.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||||
|
let targets = build_tier_mutation_affected_targets(
|
||||||
|
TierMutationIntentKind::Add,
|
||||||
|
HashSet::from(["COLD-A".to_string()]),
|
||||||
|
&previous,
|
||||||
|
&candidate,
|
||||||
|
)
|
||||||
|
.expect("added tier should have a target");
|
||||||
|
let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &candidate, targets)
|
||||||
|
.expect("intent should build")
|
||||||
|
.expect("add should require an intent");
|
||||||
|
intent.expires_at_unix_nanos = 1;
|
||||||
|
candidate
|
||||||
|
.save_tiering_config_if_current(store.clone(), None)
|
||||||
|
.await
|
||||||
|
.expect("candidate should persist");
|
||||||
|
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||||
|
.await
|
||||||
|
.expect("peer prepare should persist");
|
||||||
|
|
||||||
|
let manager = TierConfigMgr::new();
|
||||||
|
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
TIER_MUTATION_TEST_PEERS
|
||||||
|
.scope(vec![FakeTierMutationPeer::boxed("peer-a", calls.clone(), Err("offline"))], async {
|
||||||
|
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||||
|
.await
|
||||||
|
.expect_err("failed peer commit must leave recovery incomplete");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let retained = TierConfigMgr::load_tier_mutation_intents(store.clone())
|
||||||
|
.await
|
||||||
|
.expect("peer scan should succeed");
|
||||||
|
assert_eq!(retained.len(), 1);
|
||||||
|
assert_eq!(retained[0].state, TierMutationIntentState::Committed);
|
||||||
|
let error = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||||
|
Ok(_) => panic!("failed peer commit must retain the fence"),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
assert!(error.message.contains("being replaced"), "{error}");
|
||||||
|
|
||||||
|
TIER_MUTATION_TEST_PEERS
|
||||||
|
.scope(
|
||||||
|
vec![FakeTierMutationPeer::boxed(
|
||||||
|
"peer-a",
|
||||||
|
calls.clone(),
|
||||||
|
Ok(PeerTierMutationState::Committed),
|
||||||
|
)],
|
||||||
|
async {
|
||||||
|
TIER_DRIVER_TEST_FACTORY
|
||||||
|
.scope(healthy_driver_factory(), async {
|
||||||
|
TierConfigMgr::reload_handle_with(&manager, store.clone())
|
||||||
|
.await
|
||||||
|
.expect("peer commit retry should recover");
|
||||||
|
TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||||
|
.await
|
||||||
|
.expect("tier should be readable after the retry");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(lock_unpoisoned(&calls).len(), 2, "recovery should retry the peer commit");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn coordinator_mutation_recovery_scan_blocks_new_operation_lease() {
|
async fn coordinator_mutation_recovery_scan_blocks_new_operation_lease() {
|
||||||
let store = Arc::new(CasConfigStore::default());
|
let store = Arc::new(CasConfigStore::default());
|
||||||
|
|||||||
@@ -527,7 +527,7 @@ impl ECStore {
|
|||||||
async fn acquire_heal_format_fence(
|
async fn acquire_heal_format_fence(
|
||||||
&self,
|
&self,
|
||||||
) -> Result<(
|
) -> Result<(
|
||||||
tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
|
crate::core::pools::PoolMetaSaveGateWriteGuard<'_>,
|
||||||
NamespaceLockGuard,
|
NamespaceLockGuard,
|
||||||
NamespaceLockGuard,
|
NamespaceLockGuard,
|
||||||
PoolMeta,
|
PoolMeta,
|
||||||
@@ -538,7 +538,7 @@ impl ECStore {
|
|||||||
.first()
|
.first()
|
||||||
.cloned()
|
.cloned()
|
||||||
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
|
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
|
||||||
let mut write_state = self.pool_meta_save_gate.lock().await;
|
let mut write_state = self.pool_meta_save_gate.write().await;
|
||||||
write_state.ensure_write_safe("heal format fence failed")?;
|
write_state.ensure_write_safe("heal format fence failed")?;
|
||||||
|
|
||||||
// Metadata fence order is part of the decommission/rebalance protocol:
|
// Metadata fence order is part of the decommission/rebalance protocol:
|
||||||
@@ -1079,7 +1079,7 @@ impl ECStore {
|
|||||||
pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||||
let _movement_guard = movement_gate.read().await;
|
let _movement_guard = movement_gate.read().await;
|
||||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
let save_guard = self.pool_meta_save_gate.write().await;
|
||||||
save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
|
save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
|
||||||
let mut fenced_pools = BTreeSet::new();
|
let mut fenced_pools = BTreeSet::new();
|
||||||
{
|
{
|
||||||
@@ -2527,7 +2527,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
}
|
}
|
||||||
@@ -3550,7 +3550,11 @@ mod tests {
|
|||||||
async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
|
async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
|
||||||
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||||
remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||||
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
store
|
||||||
|
.pool_meta_save_gate
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.observe_replicas(PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
repair_write_safe: false,
|
repair_write_safe: false,
|
||||||
});
|
});
|
||||||
@@ -3688,7 +3692,7 @@ mod tests {
|
|||||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||||
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
||||||
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
|
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
|
||||||
let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
|
let recovery_latch = store.pool_meta_save_gate.write().await.aborted_transaction_latch_for_test();
|
||||||
let mut heal = tokio::spawn({
|
let mut heal = tokio::spawn({
|
||||||
let store = Arc::clone(&store);
|
let store = Arc::clone(&store);
|
||||||
async move { store.handle_heal_format(false).await }
|
async move { store.handle_heal_format(false).await }
|
||||||
@@ -3848,7 +3852,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
let state = source.pool_meta_save_gate.write().await.independent_clone_for_test();
|
||||||
Arc::new(ECStore {
|
Arc::new(ECStore {
|
||||||
id: Uuid::new_v4(),
|
id: Uuid::new_v4(),
|
||||||
disk_map: Default::default(),
|
disk_map: Default::default(),
|
||||||
@@ -3858,7 +3862,7 @@ mod tests {
|
|||||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||||
start_gate: tokio::sync::Mutex::new(()),
|
start_gate: tokio::sync::Mutex::new(()),
|
||||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(state),
|
||||||
ctx: Arc::clone(&source.ctx),
|
ctx: Arc::clone(&source.ctx),
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
})
|
})
|
||||||
@@ -4040,7 +4044,7 @@ mod tests {
|
|||||||
let missing = remove_pool_meta_shard(&store, 0).await;
|
let missing = remove_pool_meta_shard(&store, 0).await;
|
||||||
let (guard, admissions, mut state) = store.acquire_pool_meta_object_heal_fence(&[0]).await.expect("heal authority");
|
let (guard, admissions, mut state) = store.acquire_pool_meta_object_heal_fence(&[0]).await.expect("heal authority");
|
||||||
assert!(admissions.into_iter().all(|admission| admission.is_ok()));
|
assert!(admissions.into_iter().all(|admission| admission.is_ok()));
|
||||||
assert!(store.pool_meta_save_gate.try_lock().is_err(), "heal must retain the earlier save gate");
|
assert!(store.pool_meta_save_gate.try_write().is_err(), "heal must retain the earlier save gate");
|
||||||
let writer_store = Arc::clone(&store);
|
let writer_store = Arc::clone(&store);
|
||||||
let writer = tokio::spawn(async move { writer_store.save_current_pool_meta_for_test(&[0]).await });
|
let writer = tokio::spawn(async move { writer_store.save_current_pool_meta_for_test(&[0]).await });
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
@@ -4687,7 +4691,11 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
|
async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
|
||||||
let store = minimal_heal_store().await;
|
let store = minimal_heal_store().await;
|
||||||
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
|
store
|
||||||
|
.pool_meta_save_gate
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.observe_replicas(PoolMetaReplicaState {
|
||||||
needs_repair: true,
|
needs_repair: true,
|
||||||
repair_write_safe: false,
|
repair_write_safe: false,
|
||||||
});
|
});
|
||||||
@@ -4724,7 +4732,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("peer dispatch should start");
|
.expect("peer dispatch should start");
|
||||||
assert!(
|
assert!(
|
||||||
store.pool_meta_save_gate.try_lock().is_ok(),
|
store.pool_meta_save_gate.try_write().is_ok(),
|
||||||
"coordinator must release its local save gate before waiting for peers"
|
"coordinator must release its local save gate before waiting for peers"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -5116,7 +5124,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
};
|
};
|
||||||
@@ -5136,7 +5144,7 @@ mod tests {
|
|||||||
.save_for_startup(store.pools.clone())
|
.save_for_startup(store.pools.clone())
|
||||||
.await
|
.await
|
||||||
.expect("pool metadata should be persisted before format heal");
|
.expect("pool metadata should be persisted before format heal");
|
||||||
*store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
|
*store.pool_meta_save_gate.write().await = crate::core::pools::PoolMetaWriteState::default();
|
||||||
|
|
||||||
let replacement_endpoint = store.pools[1].endpoints.endpoints.as_ref()[2].clone();
|
let replacement_endpoint = store.pools[1].endpoints.endpoints.as_ref()[2].clone();
|
||||||
tokio::fs::remove_file(
|
tokio::fs::remove_file(
|
||||||
|
|||||||
@@ -676,7 +676,7 @@ impl ECStore {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers,
|
decommission_cancelers,
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::new(pool_meta_write_state),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(pool_meta_write_state),
|
||||||
// Adopt the caller's context (the process bootstrap one on the
|
// Adopt the caller's context (the process bootstrap one on the
|
||||||
// legacy path) so startup writes (erasure type recorded before
|
// legacy path) so startup writes (erasure type recorded before
|
||||||
// this point) and later reads share one cell.
|
// this point) and later reads share one cell.
|
||||||
@@ -725,7 +725,7 @@ impl ECStore {
|
|||||||
// Lock order is save gate -> distributed metadata writer, matching
|
// Lock order is save gate -> distributed metadata writer, matching
|
||||||
// runtime mutations. RPC locks are available before ECStore publication
|
// runtime mutations. RPC locks are available before ECStore publication
|
||||||
// and do not depend on the later fleet proof.
|
// and do not depend on the later fleet proof.
|
||||||
let mut startup_write_state = self.pool_meta_save_gate.lock().await;
|
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||||
let pool = self
|
let pool = self
|
||||||
.pools
|
.pools
|
||||||
|
|||||||
@@ -127,28 +127,30 @@ impl AuthoritativeListOperations for ECStore {
|
|||||||
marker: Option<String>,
|
marker: Option<String>,
|
||||||
max_keys: i32,
|
max_keys: i32,
|
||||||
) -> Result<ListObjectVersionsInfo> {
|
) -> Result<ListObjectVersionsInfo> {
|
||||||
if max_keys <= 0 || max_keys > MAX_OBJECT_LIST {
|
self.list_unversioned_authoritative_page(bucket, prefix, marker, max_keys, None)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_identifier_tree_leaves_authoritative(
|
||||||
|
self: Arc<Self>,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: &str,
|
||||||
|
leaf: &str,
|
||||||
|
marker: Option<String>,
|
||||||
|
max_keys: i32,
|
||||||
|
) -> Result<ListObjectVersionsInfo> {
|
||||||
|
let valid_prefix = prefix
|
||||||
|
.strip_suffix('/')
|
||||||
|
.is_some_and(|root| !root.is_empty() && root.split('/').all(|part| !part.is_empty() && !matches!(part, "." | "..")));
|
||||||
|
if !valid_prefix || !crate::disk::is_identifier_tree_leaf(leaf) {
|
||||||
return Err(StorageError::InvalidArgument(
|
return Err(StorageError::InvalidArgument(
|
||||||
bucket.to_string(),
|
bucket.to_string(),
|
||||||
prefix.to_string(),
|
prefix.to_string(),
|
||||||
"authoritative page size is out of range".to_string(),
|
"identifier-tree listing requires a directory prefix and a dotted leaf name".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let opts = ListPathOptions {
|
self.list_unversioned_authoritative_page(bucket, prefix, marker, max_keys, Some(leaf.to_string()))
|
||||||
bucket: bucket.to_string(),
|
.await
|
||||||
prefix: prefix.to_string(),
|
|
||||||
marker: marker.clone(),
|
|
||||||
limit: max_keys,
|
|
||||||
recursive: true,
|
|
||||||
incl_deleted: true,
|
|
||||||
versioned: true,
|
|
||||||
authoritative: true,
|
|
||||||
caller: ListObjectsGatherCaller::S3ListVersions,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let mut page = self.list_path(&opts).await?;
|
|
||||||
page.entries.forward_past(marker);
|
|
||||||
unversioned_authoritative_page(bucket, page)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn walk_authoritative(
|
async fn walk_authoritative(
|
||||||
@@ -436,6 +438,8 @@ pub struct ListPathOptions {
|
|||||||
// Internal bounded pages require GET-equivalent per-set resolution.
|
// Internal bounded pages require GET-equivalent per-set resolution.
|
||||||
pub authoritative: bool,
|
pub authoritative: bool,
|
||||||
|
|
||||||
|
pub identifier_tree_leaf: Option<String>,
|
||||||
|
|
||||||
pub stop_disk_at_limit: bool,
|
pub stop_disk_at_limit: bool,
|
||||||
|
|
||||||
pub pool_idx: Option<usize>,
|
pub pool_idx: Option<usize>,
|
||||||
@@ -1101,6 +1105,39 @@ impl ListPathOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ECStore {
|
impl ECStore {
|
||||||
|
async fn list_unversioned_authoritative_page(
|
||||||
|
self: Arc<Self>,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: &str,
|
||||||
|
marker: Option<String>,
|
||||||
|
max_keys: i32,
|
||||||
|
identifier_tree_leaf: Option<String>,
|
||||||
|
) -> Result<ListObjectVersionsInfo> {
|
||||||
|
if max_keys <= 0 || max_keys > MAX_OBJECT_LIST {
|
||||||
|
return Err(StorageError::InvalidArgument(
|
||||||
|
bucket.to_string(),
|
||||||
|
prefix.to_string(),
|
||||||
|
"authoritative page size is out of range".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let opts = ListPathOptions {
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
prefix: prefix.to_string(),
|
||||||
|
marker: marker.clone(),
|
||||||
|
limit: max_keys,
|
||||||
|
recursive: true,
|
||||||
|
incl_deleted: true,
|
||||||
|
versioned: true,
|
||||||
|
authoritative: true,
|
||||||
|
identifier_tree_leaf,
|
||||||
|
caller: ListObjectsGatherCaller::S3ListVersions,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut page = self.list_path(&opts).await?;
|
||||||
|
page.entries.forward_past(marker);
|
||||||
|
unversioned_authoritative_page(bucket, page)
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
// @continuation_token marker
|
// @continuation_token marker
|
||||||
// @start_after as marker when continuation_token empty
|
// @start_after as marker when continuation_token empty
|
||||||
@@ -2936,6 +2973,33 @@ impl SetDisks {
|
|||||||
Ok(page)
|
Ok(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_authoritative_listed_entry(
|
||||||
|
bucket: &str,
|
||||||
|
selection: Option<(&str, &str)>,
|
||||||
|
entries: MetaCacheEntries,
|
||||||
|
errors: &[Option<DiskError>],
|
||||||
|
disk_count: usize,
|
||||||
|
parity: usize,
|
||||||
|
) -> crate::disk::error::Result<Option<MetaCacheEntry>> {
|
||||||
|
if disk_count == 0 || entries.0.len() > disk_count || entries.0.len() != errors.len() {
|
||||||
|
return Err(DiskError::ErasureReadQuorum);
|
||||||
|
}
|
||||||
|
// An old peer may ignore the optional disk selector. Its noncandidate
|
||||||
|
// rows must be ignored before metadata parsing. Selected keys retain
|
||||||
|
// the original resolver and all of its quorum/corruption checks.
|
||||||
|
if selection.is_some_and(|(prefix, leaf)| {
|
||||||
|
entries
|
||||||
|
.0
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.next()
|
||||||
|
.is_some_and(|entry| !crate::disk::identifier_tree_leaf_matches(prefix, leaf, &entry.name))
|
||||||
|
}) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Self::resolve_listed_versions(bucket, entries, errors, disk_count, parity)
|
||||||
|
}
|
||||||
|
|
||||||
/// Version scan for set-scoped callers that act on absence: set walks
|
/// Version scan for set-scoped callers that act on absence: set walks
|
||||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||||
/// recursive prefix-delete scan. Every online drive is read and each
|
/// recursive prefix-delete scan. Every online drive is read and each
|
||||||
@@ -2959,6 +3023,8 @@ impl SetDisks {
|
|||||||
let first_error = Arc::new(tokio::sync::Mutex::new(None));
|
let first_error = Arc::new(tokio::sync::Mutex::new(None));
|
||||||
let error_sink = Arc::clone(&first_error);
|
let error_sink = Arc::clone(&first_error);
|
||||||
let bucket = opts.bucket.clone();
|
let bucket = opts.bucket.clone();
|
||||||
|
let prefix = opts.prefix.clone();
|
||||||
|
let identifier_tree_leaf = opts.identifier_tree_leaf.clone();
|
||||||
let cancel = rx.clone();
|
let cancel = rx.clone();
|
||||||
let result = list_path_raw_with_claim_tracker(
|
let result = list_path_raw_with_claim_tracker(
|
||||||
rx,
|
rx,
|
||||||
@@ -2976,6 +3042,7 @@ impl SetDisks {
|
|||||||
forward_to: opts.marker,
|
forward_to: opts.marker,
|
||||||
min_disks: read_quorum,
|
min_disks: read_quorum,
|
||||||
preserve_replica_metadata: true,
|
preserve_replica_metadata: true,
|
||||||
|
identifier_tree_leaf: opts.identifier_tree_leaf,
|
||||||
// A reader's local page may be consumed by stale entries. Only
|
// A reader's local page may be consumed by stale entries. Only
|
||||||
// the resolved, merged page may stop the authoritative walk.
|
// the resolved, merged page may stop the authoritative walk.
|
||||||
per_disk_limit: 0,
|
per_disk_limit: 0,
|
||||||
@@ -2983,7 +3050,14 @@ impl SetDisks {
|
|||||||
walkdir_timeout: opts.walkdir_timeout,
|
walkdir_timeout: opts.walkdir_timeout,
|
||||||
walkdir_stall_timeout: opts.walkdir_stall_timeout,
|
walkdir_stall_timeout: opts.walkdir_stall_timeout,
|
||||||
partial: Some(Box::new(move |entries, errors| {
|
partial: Some(Box::new(move |entries, errors| {
|
||||||
let resolved = Self::resolve_listed_versions(&bucket, entries, errors, disk_count, parity);
|
let resolved = Self::resolve_authoritative_listed_entry(
|
||||||
|
&bucket,
|
||||||
|
identifier_tree_leaf.as_deref().map(|leaf| (prefix.as_str(), leaf)),
|
||||||
|
entries,
|
||||||
|
errors,
|
||||||
|
disk_count,
|
||||||
|
parity,
|
||||||
|
);
|
||||||
let sender = sender.clone();
|
let sender = sender.clone();
|
||||||
let cancel = cancel.clone();
|
let cancel = cancel.clone();
|
||||||
let first_error = Arc::clone(&error_sink);
|
let first_error = Arc::clone(&error_sink);
|
||||||
@@ -6997,6 +7071,155 @@ mod test {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn authoritative_identifier_tree_filters_old_peer_rows_before_metadata_resolution() {
|
||||||
|
let prefix = "table-catalog/namespaces/";
|
||||||
|
let leaf = "table-entry.json";
|
||||||
|
let corrupt = |name: &str| {
|
||||||
|
MetaCacheEntries(vec![
|
||||||
|
Some(MetaCacheEntry {
|
||||||
|
name: format!("{prefix}{name}"),
|
||||||
|
metadata: b"corrupt metadata".to_vec(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
4
|
||||||
|
])
|
||||||
|
};
|
||||||
|
let report = "sales/tables/orders/maintenance/hash/jobs/job.json";
|
||||||
|
let errors = vec![None; 4];
|
||||||
|
assert!(
|
||||||
|
SetDisks::resolve_authoritative_listed_entry(RUSTFS_META_BUCKET, None, corrupt(report), &errors, 4, 2,).is_err(),
|
||||||
|
"the control must actually parse and reject the corrupt metadata"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
SetDisks::resolve_authoritative_listed_entry(
|
||||||
|
RUSTFS_META_BUCKET,
|
||||||
|
Some((prefix, leaf)),
|
||||||
|
corrupt(report),
|
||||||
|
&errors,
|
||||||
|
4,
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
.expect("an old peer's unrelated report is outside the selected scan")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
SetDisks::resolve_authoritative_listed_entry(
|
||||||
|
RUSTFS_META_BUCKET,
|
||||||
|
Some((prefix, leaf)),
|
||||||
|
corrupt("sales/tables/orders/maintenance/tables/owner/table-entry.json"),
|
||||||
|
&errors,
|
||||||
|
4,
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
.is_err(),
|
||||||
|
"a matching nested owner must still fail closed on corruption"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authoritative_identifier_tree_pages_preserve_nested_owners_and_read_quorum() {
|
||||||
|
let (dirs, store) = single_set_store(4).await;
|
||||||
|
let prefix = "table-catalog/identifier-tree/";
|
||||||
|
let leaf = "table-entry.json";
|
||||||
|
for (root, selected) in [
|
||||||
|
("table-catalog/identifier-tree", leaf),
|
||||||
|
(prefix, "../entry.json"),
|
||||||
|
(prefix, "table-entry"),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, root, selected, None, 2)
|
||||||
|
.await,
|
||||||
|
Err(StorageError::InvalidArgument(..))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let names = [
|
||||||
|
format!("{prefix}a/tables/first/{leaf}"),
|
||||||
|
format!("{prefix}sales/tables/orders/maintenance/tables/owner/{leaf}"),
|
||||||
|
format!("{prefix}z/tables/last/{leaf}"),
|
||||||
|
];
|
||||||
|
for name in &names {
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, name, |slot| Some(unversioned_catalog_metadata(name, slot))).await;
|
||||||
|
}
|
||||||
|
for name in [
|
||||||
|
format!("{prefix}sales"),
|
||||||
|
format!("{prefix}sales/tables/orders/maintenance/hash/jobs/job.json"),
|
||||||
|
] {
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &name, |_| Some(b"corrupt noncandidate metadata".to_vec())).await;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.clone()
|
||||||
|
.list_unversioned_objects_authoritative(RUSTFS_META_BUCKET, prefix, None, 100)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"the ordinary authoritative scan must expose the corrupt control objects"
|
||||||
|
);
|
||||||
|
let first = store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||||
|
.await
|
||||||
|
.expect("noncandidate metadata must not be read");
|
||||||
|
assert!(first.is_truncated);
|
||||||
|
assert_eq!(
|
||||||
|
first.objects.iter().map(|object| &object.name).collect::<Vec<_>>(),
|
||||||
|
names[..2].iter().collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
let second = store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, first.next_marker, 2)
|
||||||
|
.await
|
||||||
|
.expect("the next page must retain the final owner");
|
||||||
|
assert!(!second.is_truncated);
|
||||||
|
assert_eq!(second.objects.iter().map(|object| &object.name).collect::<Vec<_>>(), vec![&names[2]]);
|
||||||
|
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |_| Some(b"corrupt selected metadata".to_vec())).await;
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"corruption at a selected key cannot be skipped to fill the page"
|
||||||
|
);
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||||
|
Some(unversioned_catalog_metadata(&names[0], slot))
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||||
|
Some(versioned_object_metadata(RUSTFS_META_BUCKET, &names[0], &[(2, 1_700_000_002)], slot).to_vec())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"a versioned ownership entry cannot be accepted as an unversioned proof"
|
||||||
|
);
|
||||||
|
write_version_replicas(&dirs, RUSTFS_META_BUCKET, &names[0], |slot| {
|
||||||
|
Some(unversioned_catalog_metadata(&names[0], slot))
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let set = &store.pools[0].disk_set[0];
|
||||||
|
for disk in set.disks.write().await.iter_mut().skip(1) {
|
||||||
|
*disk = None;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(RUSTFS_META_BUCKET, prefix, leaf, None, 2)
|
||||||
|
.await,
|
||||||
|
Err(StorageError::ErasureReadQuorum)
|
||||||
|
),
|
||||||
|
"unavailable replicas cannot prove owner absence"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||||
let bucket = "catalog-cleanup-versions";
|
let bucket = "catalog-cleanup-versions";
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ use crate::bucket::utils::check_put_object_part_args;
|
|||||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||||
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
|
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
|
||||||
use crate::config::storageclass;
|
use crate::config::storageclass;
|
||||||
use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaWriteState};
|
use crate::core::pools::{DecommissionCanceler, PoolMeta, PoolMetaSaveGate};
|
||||||
use crate::disk::endpoint::{Endpoint, EndpointType};
|
use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||||
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
@@ -480,9 +480,13 @@ pub struct ECStore {
|
|||||||
/// Serializes full-document pool metadata saves and retains a fail-closed
|
/// Serializes full-document pool metadata saves and retains a fail-closed
|
||||||
/// write block after startup observes an unreadable replica.
|
/// write block after startup observes an unreadable replica.
|
||||||
///
|
///
|
||||||
|
/// Immutable admission proofs share authority; publication, recovery and
|
||||||
|
/// authority changes retain exclusive access through their existing I/O.
|
||||||
|
/// Each writer publishes its final state on release, and health
|
||||||
|
/// inspection reads that committed state without acquiring the gate.
|
||||||
/// Lock order: acquire `pool_meta_save_gate`, then the distributed
|
/// Lock order: acquire `pool_meta_save_gate`, then the distributed
|
||||||
/// `pool.bin` fence, then clone `pool_meta` under a short read lock.
|
/// `pool.bin` fence, then clone `pool_meta` under a short read lock.
|
||||||
pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
|
pub(crate) pool_meta_save_gate: PoolMetaSaveGate,
|
||||||
/// Per-instance runtime state (Phase 5, backlog#939).
|
/// Per-instance runtime state (Phase 5, backlog#939).
|
||||||
///
|
///
|
||||||
/// Carries this instance's identity/runtime out of the process globals so
|
/// Carries this instance's identity/runtime out of the process globals so
|
||||||
@@ -555,7 +559,6 @@ impl Default for ScannerDataMovementPauseStatus {
|
|||||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||||
pub struct PoolMetaWriteGateStatus {
|
pub struct PoolMetaWriteGateStatus {
|
||||||
pub writes_ready: bool,
|
pub writes_ready: bool,
|
||||||
pub check_timed_out: bool,
|
|
||||||
pub write_blocked: bool,
|
pub write_blocked: bool,
|
||||||
pub transaction_aborted: bool,
|
pub transaction_aborted: bool,
|
||||||
pub pool_meta_absent: bool,
|
pub pool_meta_absent: bool,
|
||||||
@@ -571,7 +574,6 @@ impl Default for PoolMetaWriteGateStatus {
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
writes_ready: true,
|
writes_ready: true,
|
||||||
check_timed_out: false,
|
|
||||||
write_blocked: false,
|
write_blocked: false,
|
||||||
transaction_aborted: false,
|
transaction_aborted: false,
|
||||||
pool_meta_absent: false,
|
pool_meta_absent: false,
|
||||||
@@ -1936,7 +1938,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
};
|
};
|
||||||
@@ -2011,7 +2013,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: Arc::default(),
|
bucket_fence_registry: Arc::default(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -2133,7 +2133,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10003,7 +10003,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
}
|
}
|
||||||
@@ -10095,7 +10095,7 @@ mod tests {
|
|||||||
rebalance_meta: RwLock::new(None),
|
rebalance_meta: RwLock::new(None),
|
||||||
decommission_cancelers: RwLock::new(Vec::new()),
|
decommission_cancelers: RwLock::new(Vec::new()),
|
||||||
start_gate: Mutex::new(()),
|
start_gate: Mutex::new(()),
|
||||||
pool_meta_save_gate: Mutex::default(),
|
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
|
||||||
ctx,
|
ctx,
|
||||||
bucket_fence_registry: std::sync::Arc::default(),
|
bucket_fence_registry: std::sync::Arc::default(),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ pub(crate) fn notify_configuration_hint() -> String {
|
|||||||
format!(
|
format!(
|
||||||
"No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
|
"No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
|
||||||
rustfs_config::ENV_NOTIFY_ENABLE,
|
rustfs_config::ENV_NOTIFY_ENABLE,
|
||||||
rustfs_targets::store::resolve_queue_dir(rustfs_config::EVENT_DEFAULT_DIR),
|
rustfs_config::EVENT_DEFAULT_DIR,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -64,6 +64,8 @@ async-compression = { workspace = true, features = ["tokio", "gzip", "bzip2"] }
|
|||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
arc-swap.workspace = true
|
arc-swap.workspace = true
|
||||||
bytes = { workspace = true, features = ["serde"] }
|
bytes = { workspace = true, features = ["serde"] }
|
||||||
|
base64-simd.workspace = true
|
||||||
|
flatbuffers.workspace = true
|
||||||
chrono = { workspace = true, features = ["serde"] }
|
chrono = { workspace = true, features = ["serde"] }
|
||||||
crc-fast.workspace = true
|
crc-fast.workspace = true
|
||||||
rustfs-common.workspace = true
|
rustfs-common.workspace = true
|
||||||
@@ -75,6 +77,7 @@ futures-core = { workspace = true }
|
|||||||
flate2.workspace = true
|
flate2.workspace = true
|
||||||
brotli.workspace = true
|
brotli.workspace = true
|
||||||
lz4.workspace = true
|
lz4.workspace = true
|
||||||
|
lz4_flex.workspace = true
|
||||||
http.workspace = true
|
http.workspace = true
|
||||||
memchr.workspace = true
|
memchr.workspace = true
|
||||||
s3s = { workspace = true, features = ["minio"] }
|
s3s = { workspace = true, features = ["minio"] }
|
||||||
|
|||||||
@@ -221,7 +221,7 @@ struct ThriftCompactCursor<'a> {
|
|||||||
remaining: usize,
|
remaining: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ThriftCompactCursor<'_> {
|
impl<'a> ThriftCompactCursor<'a> {
|
||||||
fn error() -> ParquetBudgetError {
|
fn error() -> ParquetBudgetError {
|
||||||
ParquetBudgetError::Invalid("Parquet thrift metadata is malformed".to_owned())
|
ParquetBudgetError::Invalid("Parquet thrift metadata is malformed".to_owned())
|
||||||
}
|
}
|
||||||
@@ -241,6 +241,14 @@ impl ThriftCompactCursor<'_> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn binary(&mut self) -> Result<&'a [u8]> {
|
||||||
|
let len = usize::try_from(self.varint()?).map_err(|_| Self::error())?;
|
||||||
|
self.charge(len.checked_mul(2).ok_or_else(budget_error)?)?;
|
||||||
|
let start = self.pos;
|
||||||
|
self.skip_bytes(len)?;
|
||||||
|
Ok(&self.bytes[start..self.pos])
|
||||||
|
}
|
||||||
|
|
||||||
fn varint(&mut self) -> Result<u64> {
|
fn varint(&mut self) -> Result<u64> {
|
||||||
let mut value = 0_u64;
|
let mut value = 0_u64;
|
||||||
for shift in (0..64).step_by(7) {
|
for shift in (0..64).step_by(7) {
|
||||||
@@ -335,11 +343,7 @@ impl ThriftCompactCursor<'_> {
|
|||||||
THRIFT_COMPACT_BYTE => self.skip_bytes(1),
|
THRIFT_COMPACT_BYTE => self.skip_bytes(1),
|
||||||
THRIFT_COMPACT_I16 | THRIFT_COMPACT_I32 | THRIFT_COMPACT_I64 => self.varint().map(drop),
|
THRIFT_COMPACT_I16 | THRIFT_COMPACT_I32 | THRIFT_COMPACT_I64 => self.varint().map(drop),
|
||||||
THRIFT_COMPACT_DOUBLE => self.skip_bytes(8),
|
THRIFT_COMPACT_DOUBLE => self.skip_bytes(8),
|
||||||
THRIFT_COMPACT_BINARY => {
|
THRIFT_COMPACT_BINARY => self.binary().map(drop),
|
||||||
let len = usize::try_from(self.varint()?).map_err(|_| Self::error())?;
|
|
||||||
self.charge(len.checked_mul(2).ok_or_else(budget_error)?)?;
|
|
||||||
self.skip_bytes(len)
|
|
||||||
}
|
|
||||||
THRIFT_COMPACT_LIST | THRIFT_COMPACT_SET => {
|
THRIFT_COMPACT_LIST | THRIFT_COMPACT_SET => {
|
||||||
let (element, count) = self.list_header()?;
|
let (element, count) = self.list_header()?;
|
||||||
for _ in 0..count {
|
for _ in 0..count {
|
||||||
@@ -586,21 +590,98 @@ impl ThriftCompactCursor<'_> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn arrow_schema_metadata(&mut self) -> Result<Option<&'a [u8]>> {
|
||||||
|
let (element, count) = self.list_header()?;
|
||||||
|
if element != THRIFT_COMPACT_STRUCT {
|
||||||
|
return Err(Self::error());
|
||||||
|
}
|
||||||
|
let mut arrow_schema = None;
|
||||||
|
for _ in 0..count {
|
||||||
|
let mut field = 0;
|
||||||
|
let mut key = None;
|
||||||
|
let mut value = None;
|
||||||
|
loop {
|
||||||
|
let header = self.byte()?;
|
||||||
|
if header == THRIFT_COMPACT_STOP {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
field = self.field_id(field, header)?;
|
||||||
|
let kind = header & 0x0f;
|
||||||
|
if matches!(field, 1 | 2) && kind == THRIFT_COMPACT_BINARY {
|
||||||
|
let bytes = self.binary()?;
|
||||||
|
if field == 1 {
|
||||||
|
key = Some(bytes);
|
||||||
|
} else {
|
||||||
|
value = Some(bytes);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.skip_field(MetadataStruct::KeyValue, field, kind, 1)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Match the reader's last key with a value; absent values are ignored.
|
||||||
|
if key == Some(b"ARROW:schema".as_slice()) && value.is_some() {
|
||||||
|
arrow_schema = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(arrow_schema)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_arrow_schema(&mut self, encoded: &[u8]) -> Result<()> {
|
||||||
|
use datafusion::arrow::ipc;
|
||||||
|
let invalid = || ParquetBudgetError::Invalid("Parquet Arrow schema metadata is malformed".to_owned());
|
||||||
|
// This reservation precedes base64 allocation. The binary field has
|
||||||
|
// already reserved its owned metadata copies; decoding needs another buffer.
|
||||||
|
self.charge(encoded.len())?;
|
||||||
|
let bytes = base64_simd::STANDARD.decode_to_vec(encoded).map_err(|_| invalid())?;
|
||||||
|
let message = if bytes.len() > 8 && bytes.starts_with(&[255; 4]) {
|
||||||
|
&bytes[8..]
|
||||||
|
} else {
|
||||||
|
bytes.as_slice()
|
||||||
|
};
|
||||||
|
// The verifier visits shared offsets for every occurrence, before the
|
||||||
|
// reader expands them into owned fields and metadata. Split the remaining
|
||||||
|
// allowance between string/buffer copies and per-table Rust containers:
|
||||||
|
// four owned copies per apparent byte and 1 KiB per visited table.
|
||||||
|
let defaults = flatbuffers::VerifierOptions::default();
|
||||||
|
let options = flatbuffers::VerifierOptions {
|
||||||
|
max_apparent_size: defaults.max_apparent_size.min(self.remaining / 8),
|
||||||
|
max_tables: defaults.max_tables.min(self.remaining / (2 * 1024)),
|
||||||
|
..defaults
|
||||||
|
};
|
||||||
|
let message = ipc::root_as_message_with_opts(&options, message).map_err(|error| match error {
|
||||||
|
flatbuffers::InvalidFlatbuffer::TooManyTables
|
||||||
|
| flatbuffers::InvalidFlatbuffer::ApparentSizeTooLarge
|
||||||
|
| flatbuffers::InvalidFlatbuffer::DepthLimitReached => budget_error(),
|
||||||
|
_ => invalid(),
|
||||||
|
})?;
|
||||||
|
let schema = message.header_as_schema().ok_or_else(invalid)?;
|
||||||
|
if schema.fields().is_none() {
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Walk the footer's `FileMetaData`. The pinned parquet reserves the
|
/// Walk the footer's `FileMetaData`. The pinned parquet reserves the
|
||||||
/// declared `row_groups` capacity before reading one element; charge that
|
/// declared `row_groups` capacity before reading one element; charge that
|
||||||
/// reservation to the budget so a small footer cannot request gigabytes.
|
/// reservation to the budget so a small footer cannot request gigabytes.
|
||||||
fn file_metadata(&mut self, limit: usize) -> Result<()> {
|
fn file_metadata(&mut self, limit: usize) -> Result<()> {
|
||||||
let mut field_id = 0_i16;
|
let mut field_id = 0_i16;
|
||||||
let mut reserved = 0_usize;
|
let mut reserved = 0_usize;
|
||||||
|
let mut arrow_schema = None;
|
||||||
loop {
|
loop {
|
||||||
let header = self.byte()?;
|
let header = self.byte()?;
|
||||||
if header == THRIFT_COMPACT_STOP {
|
if header == THRIFT_COMPACT_STOP {
|
||||||
|
if let Some(encoded) = arrow_schema {
|
||||||
|
self.validate_arrow_schema(encoded)?;
|
||||||
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let kind = header & 0x0f;
|
let kind = header & 0x0f;
|
||||||
field_id = self.field_id(field_id, header)?;
|
field_id = self.field_id(field_id, header)?;
|
||||||
if field_id == 2 && kind == THRIFT_COMPACT_LIST {
|
if field_id == 2 && kind == THRIFT_COMPACT_LIST {
|
||||||
self.schema()?;
|
self.schema()?;
|
||||||
|
} else if field_id == 5 && kind == THRIFT_COMPACT_LIST {
|
||||||
|
arrow_schema = self.arrow_schema_metadata()?;
|
||||||
} else if field_id == FILE_METADATA_FIELD_ROW_GROUPS && kind == THRIFT_COMPACT_LIST {
|
} else if field_id == FILE_METADATA_FIELD_ROW_GROUPS && kind == THRIFT_COMPACT_LIST {
|
||||||
let (element, count) = self.list_header()?;
|
let (element, count) = self.list_header()?;
|
||||||
if element != THRIFT_COMPACT_STRUCT {
|
if element != THRIFT_COMPACT_STRUCT {
|
||||||
@@ -760,7 +841,10 @@ fn validate_compressed_page(payload: &[u8], sizes: PageHeaderSizes, codec: Compr
|
|||||||
bounded_stream_size(brotli::Decompressor::new(payload, 4096), expanded)?;
|
bounded_stream_size(brotli::Decompressor::new(payload, 4096), expanded)?;
|
||||||
Ok(window)
|
Ok(window)
|
||||||
}
|
}
|
||||||
Compression::LZ4 if payload.starts_with(&[4, 34, 77, 24]) => {
|
Compression::LZ4 if payload.starts_with(&[4, 34, 77, 24]) || payload.starts_with(&[2, 33, 76, 24]) => {
|
||||||
|
let (block, linked) = if payload.starts_with(&[2, 33, 76, 24]) {
|
||||||
|
(8 * 1024 * 1024, false)
|
||||||
|
} else {
|
||||||
let descriptor = *payload.get(5).ok_or_else(ThriftCompactCursor::error)?;
|
let descriptor = *payload.get(5).ok_or_else(ThriftCompactCursor::error)?;
|
||||||
let block = match (descriptor >> 4) & 7 {
|
let block = match (descriptor >> 4) & 7 {
|
||||||
4 => 64 * 1024,
|
4 => 64 * 1024,
|
||||||
@@ -769,12 +853,22 @@ fn validate_compressed_page(payload: &[u8], sizes: PageHeaderSizes, codec: Compr
|
|||||||
7 => 4 * 1024 * 1024,
|
7 => 4 * 1024 * 1024,
|
||||||
_ => return Err(ThriftCompactCursor::error()),
|
_ => return Err(ThriftCompactCursor::error()),
|
||||||
};
|
};
|
||||||
let working = block * 2;
|
(block, payload[4] & 0x20 == 0)
|
||||||
|
};
|
||||||
|
// Match parquet's pinned fallback decoder: compressed input plus
|
||||||
|
// output, with an extra output block and 64 KiB history when linked.
|
||||||
|
let working = if linked { block * 3 + 64 * 1024 } else { block * 2 };
|
||||||
if working > limit {
|
if working > limit {
|
||||||
return Err(budget_error());
|
return Err(budget_error());
|
||||||
}
|
}
|
||||||
bounded_stream_size(lz4::Decoder::new(payload)?, expanded)?;
|
match bounded_stream_size(lz4_flex::frame::FrameDecoder::new(payload), expanded) {
|
||||||
Ok(working)
|
// Hadoop data can share the magic prefix, and parquet also
|
||||||
|
// tries bounded RAW decoding after a frame error. The bounded
|
||||||
|
// sink proved the frame produced no more than `expanded` bytes
|
||||||
|
// before that error; let the actual reader select its fallback.
|
||||||
|
Err(ParquetBudgetError::Codec(_)) => Ok(working),
|
||||||
|
result => result.map(|()| working),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_ => Ok(0),
|
_ => Ok(0),
|
||||||
}
|
}
|
||||||
@@ -851,6 +945,49 @@ mod tests {
|
|||||||
assert!(validate_compressed_page(&payload, compressed_page(&payload, 64), Compression::LZ4, 1024 * 1024).is_err());
|
assert!(validate_compressed_page(&payload, compressed_page(&payload, 64), Compression::LZ4, 1024 * 1024).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_lz4_scratch_is_admitted_before_the_fallback_decoder() {
|
||||||
|
let payload = [2, 33, 76, 24, 5, 0, 0, 0, 0x40, 42, 0, 0, 0];
|
||||||
|
let header = compressed_page(&payload, 4);
|
||||||
|
assert!(matches!(
|
||||||
|
validate_compressed_page(&payload, header, Compression::LZ4, 1024 * 1024),
|
||||||
|
Err(ParquetBudgetError::Exceeded)
|
||||||
|
));
|
||||||
|
let admitted = validate_compressed_page(&payload, header, Compression::LZ4, 32 * 1024 * 1024)
|
||||||
|
.expect("legacy LZ4 remains supported with enough scratch memory");
|
||||||
|
assert!(admitted >= 16 * 1024 * 1024);
|
||||||
|
assert!(validate_compressed_page(&payload, compressed_page(&payload, 1), Compression::LZ4, 32 * 1024 * 1024).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hadoop_lz4_can_share_the_legacy_magic_without_becoming_a_frame() {
|
||||||
|
let size = usize::try_from(u32::from_be_bytes([2, 33, 76, 24])).expect("Hadoop size");
|
||||||
|
let bytes = vec![b'x'; size];
|
||||||
|
let block = lz4_flex::block::compress(&bytes);
|
||||||
|
let mut payload = vec![2, 33, 76, 24];
|
||||||
|
payload.extend(u32::try_from(block.len()).expect("compressed block").to_be_bytes());
|
||||||
|
payload.extend(block);
|
||||||
|
validate_compressed_page(&payload, compressed_page(&payload, size), Compression::LZ4, 64 * 1024 * 1024)
|
||||||
|
.expect("Hadoop framing takes priority over the frame fallback");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn linked_lz4_accounts_for_the_history_and_both_block_buffers() {
|
||||||
|
use lz4_flex::frame::{BlockMode, BlockSize, FrameEncoder, FrameInfo};
|
||||||
|
let frame = FrameInfo::new().block_mode(BlockMode::Linked).block_size(BlockSize::Max64KB);
|
||||||
|
let mut encoder = FrameEncoder::with_frame_info(frame, Vec::new());
|
||||||
|
encoder.write_all(&vec![b'x'; 256 * 1024]).expect("linked input");
|
||||||
|
let payload = encoder.finish().expect("linked frame");
|
||||||
|
let header = compressed_page(&payload, 256 * 1024);
|
||||||
|
assert!(matches!(
|
||||||
|
validate_compressed_page(&payload, header, Compression::LZ4, 192 * 1024),
|
||||||
|
Err(ParquetBudgetError::Exceeded)
|
||||||
|
));
|
||||||
|
let admitted = validate_compressed_page(&payload, header, Compression::LZ4, 1024 * 1024)
|
||||||
|
.expect("linked frame with enough scratch memory");
|
||||||
|
assert!(admitted >= 256 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn uncompressed_v2_does_not_invoke_the_column_codec() {
|
fn uncompressed_v2_does_not_invoke_the_column_codec() {
|
||||||
let payload = b"levels-and-uncompressed-values";
|
let payload = b"levels-and-uncompressed-values";
|
||||||
@@ -941,6 +1078,170 @@ mod tests {
|
|||||||
validate_file_footer(&bytes, 1024 * 1024).expect("valid logical annotations remain accepted");
|
validate_file_footer(&bytes, 1024 * 1024).expect("valid logical annotations remain accepted");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn shared_arrow_metadata_file(entries: usize) -> Vec<u8> {
|
||||||
|
use datafusion::{
|
||||||
|
arrow::ipc,
|
||||||
|
parquet::{
|
||||||
|
file::{properties::WriterProperties, writer::SerializedFileWriter},
|
||||||
|
schema::parser::parse_message_type,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
use std::sync::Arc;
|
||||||
|
let mut builder = flatbuffers::FlatBufferBuilder::new();
|
||||||
|
let name = builder.create_string("v");
|
||||||
|
let value = builder.create_string(&"x".repeat(64 * 1024));
|
||||||
|
let data_type = ipc::Int::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::IntArgs {
|
||||||
|
bitWidth: 32,
|
||||||
|
is_signed: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let field = ipc::Field::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::FieldArgs {
|
||||||
|
name: Some(name),
|
||||||
|
nullable: true,
|
||||||
|
type_type: ipc::Type::Int,
|
||||||
|
type_: Some(data_type.as_union_value()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let fields = builder.create_vector(&[field]);
|
||||||
|
let mut metadata = Vec::new();
|
||||||
|
for n in 0..entries {
|
||||||
|
let key = builder.create_string(&format!("key-{n}"));
|
||||||
|
metadata.push(ipc::KeyValue::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::KeyValueArgs {
|
||||||
|
key: Some(key),
|
||||||
|
value: Some(value),
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let metadata = builder.create_vector(&metadata);
|
||||||
|
let schema = ipc::Schema::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::SchemaArgs {
|
||||||
|
fields: Some(fields),
|
||||||
|
custom_metadata: Some(metadata),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let message = ipc::Message::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::MessageArgs {
|
||||||
|
version: ipc::MetadataVersion::V5,
|
||||||
|
header_type: ipc::MessageHeader::Schema,
|
||||||
|
header: Some(schema.as_union_value()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
builder.finish(message, None);
|
||||||
|
let encoded = base64_simd::STANDARD.encode_to_string(builder.finished_data());
|
||||||
|
let properties = WriterProperties::builder()
|
||||||
|
.set_key_value_metadata(Some(vec![datafusion::parquet::file::metadata::KeyValue::new(
|
||||||
|
"ARROW:schema".to_owned(),
|
||||||
|
Some(encoded),
|
||||||
|
)]))
|
||||||
|
.build();
|
||||||
|
let schema = Arc::new(parse_message_type("message root { OPTIONAL INT32 v; }").expect("physical schema"));
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
SerializedFileWriter::new(&mut bytes, schema, Arc::new(properties))
|
||||||
|
.expect("writer")
|
||||||
|
.close()
|
||||||
|
.expect("footer");
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shared_arrow_metadata_is_admitted_before_owned_schema_conversion() {
|
||||||
|
use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
||||||
|
let bytes = shared_arrow_metadata_file(256);
|
||||||
|
assert!(bytes.len() < 100 * 1024, "shared offsets keep the encoded file small");
|
||||||
|
let native = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(bytes.clone())).expect("valid Arrow metadata");
|
||||||
|
assert_eq!(native.schema().metadata().len(), 256);
|
||||||
|
assert!(
|
||||||
|
matches!(validate_file_footer(&bytes, 2 * 1024 * 1024), Err(ParquetBudgetError::Exceeded)),
|
||||||
|
"32 MiB of owned metadata must not pass a 2 MiB preflight"
|
||||||
|
);
|
||||||
|
validate_file_footer(&bytes, 512 * 1024 * 1024).expect("sufficient budget preserves shared metadata");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn arrow_hint_footer(entries: &[(&[u8], Option<&[u8]>)]) -> Vec<u8> {
|
||||||
|
let count = u8::try_from(entries.len()).expect("small key/value fixture");
|
||||||
|
assert!(count < 15);
|
||||||
|
let mut bytes = vec![0x59, (count << 4) | THRIFT_COMPACT_STRUCT];
|
||||||
|
for (key, value) in entries {
|
||||||
|
bytes.push(0x18);
|
||||||
|
bytes.extend(varint(u64::try_from(key.len()).expect("key length")));
|
||||||
|
bytes.extend(*key);
|
||||||
|
if let Some(value) = value {
|
||||||
|
bytes.push(0x18);
|
||||||
|
bytes.extend(varint(u64::try_from(value.len()).expect("value length")));
|
||||||
|
bytes.extend(*value);
|
||||||
|
}
|
||||||
|
bytes.push(0);
|
||||||
|
}
|
||||||
|
bytes.push(0);
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn arrow_hint_framing_and_duplicate_keys_match_the_reader() {
|
||||||
|
use datafusion::arrow::{
|
||||||
|
datatypes::{DataType, Field, Schema},
|
||||||
|
ipc,
|
||||||
|
};
|
||||||
|
let schema = Schema::new(vec![Field::new("v", DataType::Int32, true)]);
|
||||||
|
let mut builder = flatbuffers::FlatBufferBuilder::new();
|
||||||
|
let schema = ipc::convert::schema_to_fb_offset(&mut builder, &schema);
|
||||||
|
let message = ipc::Message::create(
|
||||||
|
&mut builder,
|
||||||
|
&ipc::MessageArgs {
|
||||||
|
header_type: ipc::MessageHeader::Schema,
|
||||||
|
header: Some(schema.as_union_value()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
builder.finish(message, None);
|
||||||
|
let raw = builder.finished_data();
|
||||||
|
let mut framed = vec![255; 4];
|
||||||
|
framed.extend(u32::try_from(raw.len()).expect("small schema").to_le_bytes());
|
||||||
|
framed.extend(raw);
|
||||||
|
for message in [raw, framed.as_slice()] {
|
||||||
|
let encoded = base64_simd::STANDARD.encode_to_string(message);
|
||||||
|
let footer = arrow_hint_footer(&[
|
||||||
|
(b"ARROW:schema", Some(b"invalid")),
|
||||||
|
(b"ARROW:schema", Some(encoded.as_bytes())),
|
||||||
|
(b"ARROW:schema", None),
|
||||||
|
]);
|
||||||
|
validate_footer(&footer, 1024 * 1024).expect("last present Arrow metadata wins, both IPC framings are supported");
|
||||||
|
assert!(matches!(
|
||||||
|
validate_footer(
|
||||||
|
&arrow_hint_footer(&[
|
||||||
|
(b"ARROW:schema", Some(encoded.as_bytes())),
|
||||||
|
(b"ARROW:schema", Some(b"invalid"))
|
||||||
|
]),
|
||||||
|
1024 * 1024
|
||||||
|
),
|
||||||
|
Err(ParquetBudgetError::Invalid(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
validate_footer(&arrow_hint_footer(&[(b"other", Some(b"invalid"))]), 1024 * 1024).expect("other metadata is opaque");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn arrow_hint_value_before_key_is_still_admitted() {
|
||||||
|
// Compact struct fields are identified by ID, not required to be ordered.
|
||||||
|
let mut bytes = vec![0x59, 0x1c, 0x28, 3];
|
||||||
|
bytes.extend(b"%%%");
|
||||||
|
bytes.extend([0x08, 2, 12]);
|
||||||
|
bytes.extend(b"ARROW:schema");
|
||||||
|
bytes.extend([0, 0]);
|
||||||
|
assert!(matches!(validate_footer(&bytes, 1024 * 1024), Err(ParquetBudgetError::Invalid(_))));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn footer_collections_are_bounded_before_decoder_allocation() {
|
fn footer_collections_are_bounded_before_decoder_allocation() {
|
||||||
let mut huge = vec![0x49, 0xfc];
|
let mut huge = vec![0x49, 0xfc];
|
||||||
|
|||||||
@@ -1106,6 +1106,98 @@ mod tests {
|
|||||||
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn parquet_legacy_lz4_checks_native_scratch_before_query_decode() {
|
||||||
|
use datafusion::parquet::{basic::Compression, file::metadata::ParquetMetaDataWriter};
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)]));
|
||||||
|
let batch =
|
||||||
|
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(datafusion::arrow::array::Int32Array::from(vec![42]))])
|
||||||
|
.expect("one integer row");
|
||||||
|
let properties = datafusion::parquet::file::properties::WriterProperties::builder()
|
||||||
|
.set_dictionary_enabled(false)
|
||||||
|
.build();
|
||||||
|
let mut original = Vec::new();
|
||||||
|
let mut writer = ArrowWriter::try_new(&mut original, schema, Some(properties)).expect("writer");
|
||||||
|
writer.write(&batch).expect("row");
|
||||||
|
writer.close().expect("footer");
|
||||||
|
let metadata = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(original))
|
||||||
|
.expect("metadata")
|
||||||
|
.metadata()
|
||||||
|
.clone();
|
||||||
|
// A valid legacy frame encoding the four little-endian bytes for 42.
|
||||||
|
let payload = [2, 33, 76, 24, 5, 0, 0, 0, 0x40, 42, 0, 0, 0];
|
||||||
|
let mut page = vec![0x15, 0, 0x15, 8, 0x15, 26, 0x2c, 0x15, 2, 0x15, 0, 0x15, 6, 0x15, 6, 0, 0];
|
||||||
|
page.extend(payload);
|
||||||
|
let column = metadata
|
||||||
|
.row_group(0)
|
||||||
|
.column(0)
|
||||||
|
.clone()
|
||||||
|
.into_builder()
|
||||||
|
.set_compression(Compression::LZ4)
|
||||||
|
.set_data_page_offset(4)
|
||||||
|
.set_total_compressed_size(i64::try_from(page.len()).expect("small page"))
|
||||||
|
.build()
|
||||||
|
.expect("legacy column");
|
||||||
|
let group = metadata
|
||||||
|
.row_group(0)
|
||||||
|
.clone()
|
||||||
|
.into_builder()
|
||||||
|
.set_column_metadata(vec![column])
|
||||||
|
.build()
|
||||||
|
.expect("row group");
|
||||||
|
let metadata = metadata.as_ref().clone().into_builder().set_row_groups(vec![group]).build();
|
||||||
|
let mut encoded = b"PAR1".to_vec();
|
||||||
|
encoded.extend(page);
|
||||||
|
ParquetMetaDataWriter::new(&mut encoded, &metadata)
|
||||||
|
.finish()
|
||||||
|
.expect("legacy footer");
|
||||||
|
assert!(encoded.len() < 4096);
|
||||||
|
let mut native = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(encoded.clone()))
|
||||||
|
.expect("legacy metadata")
|
||||||
|
.build()
|
||||||
|
.expect("native reader");
|
||||||
|
let decoded = native.next().expect("one batch").expect("legacy fallback decode");
|
||||||
|
assert_eq!(
|
||||||
|
decoded
|
||||||
|
.column(0)
|
||||||
|
.as_primitive::<datafusion::arrow::datatypes::Int32Type>()
|
||||||
|
.value(0),
|
||||||
|
42
|
||||||
|
);
|
||||||
|
let store = Arc::new(InMemory::new());
|
||||||
|
store
|
||||||
|
.put(&Path::from("legacy.parquet"), Bytes::from(encoded).into())
|
||||||
|
.await
|
||||||
|
.expect("fixture");
|
||||||
|
let tight = parquet_session_with_memory(store.clone(), 2 * 1024 * 1024);
|
||||||
|
let result = ParquetSelectTable::try_new(&tight.state(), &parquet_input("legacy.parquet"), None).await;
|
||||||
|
let error = match result {
|
||||||
|
Ok(_) => panic!("legacy decoder scratch exceeds the query budget"),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
assert_eq!(error.select_error(), SelectError::ResourceExhausted);
|
||||||
|
let roomy = parquet_session_with_memory(store, 128 * 1024 * 1024);
|
||||||
|
let table = ParquetSelectTable::try_new(&roomy.state(), &parquet_input("legacy.parquet"), None)
|
||||||
|
.await
|
||||||
|
.expect("admitted legacy file");
|
||||||
|
roomy.register_table("source", table).expect("table");
|
||||||
|
let rows = roomy
|
||||||
|
.sql("SELECT v FROM source")
|
||||||
|
.await
|
||||||
|
.expect("plan")
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.expect("query");
|
||||||
|
assert_eq!(rows.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
rows[0]
|
||||||
|
.column(0)
|
||||||
|
.as_primitive::<datafusion::arrow::datatypes::Int32Type>()
|
||||||
|
.value(0),
|
||||||
|
42
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn parquet_nested_views_decode_through_the_actual_file_schema() {
|
async fn parquet_nested_views_decode_through_the_actual_file_schema() {
|
||||||
use datafusion::arrow::array::{ListArray, StringArray};
|
use datafusion::arrow::array::{ListArray, StringArray};
|
||||||
|
|||||||
@@ -591,6 +591,34 @@ pub trait AuthoritativeListOperations: ListOperations {
|
|||||||
max_keys: i32,
|
max_keys: i32,
|
||||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error>;
|
) -> Result<Self::ListObjectVersionsInfo, Self::Error>;
|
||||||
|
|
||||||
|
/// Enumerate an exact dotted leaf below identifier-like directory components
|
||||||
|
/// containing ASCII letters, digits, `_` or `-`. Noncandidate object metadata
|
||||||
|
/// is outside the scan. Namespace components may occur at any depth; reserved
|
||||||
|
/// words do not identify a subtree that can be skipped.
|
||||||
|
///
|
||||||
|
/// Returned keys retain the quorum, unversioned, EOF and cancellation
|
||||||
|
/// guarantees of `list_unversioned_objects_authoritative`. Older peers may
|
||||||
|
/// perform extra I/O; the coordinator must filter their extra rows before
|
||||||
|
/// resolving metadata. Callers fence concurrent mutations separately.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects an invalid directory prefix/leaf, invalid page size, uncertain
|
||||||
|
/// matching metadata, versioned entries or insufficient read quorum. Backends
|
||||||
|
/// without this capability return Unsupported; ordinary listing is not a fallback.
|
||||||
|
async fn list_identifier_tree_leaves_authoritative(
|
||||||
|
self: Arc<Self>,
|
||||||
|
_bucket: &str,
|
||||||
|
_prefix: &str,
|
||||||
|
_leaf: &str,
|
||||||
|
_marker: Option<String>,
|
||||||
|
_max_keys: i32,
|
||||||
|
) -> Result<Self::ListObjectVersionsInfo, Self::Error>
|
||||||
|
where
|
||||||
|
Self::Error: From<std::io::Error>,
|
||||||
|
{
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "authoritative identifier-tree listing is not supported").into())
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::TargetError;
|
use crate::TargetError;
|
||||||
use crate::store::resolve_queue_dir;
|
|
||||||
use crate::target::nats::validate_jetstream_settings;
|
use crate::target::nats::validate_jetstream_settings;
|
||||||
use crate::target::pulsar::validate_pulsar_broker;
|
use crate::target::pulsar::validate_pulsar_broker;
|
||||||
use async_nats::ServerAddr;
|
use async_nats::ServerAddr;
|
||||||
@@ -30,12 +29,6 @@ use std::path::Path;
|
|||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
/// Return the queue directory this node opens for a target, with logical defaults resolved.
|
|
||||||
pub(super) fn queue_dir_or_default(config: &KVS, key: &str, default_queue_dir: &str) -> String {
|
|
||||||
let queue_dir = config.lookup(key).unwrap_or_else(|| default_queue_dir.to_string());
|
|
||||||
resolve_queue_dir(&queue_dir).into_owned()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<String>) -> Option<(String, String)> {
|
pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<String>) -> Option<(String, String)> {
|
||||||
let normalized = rest.to_lowercase();
|
let normalized = rest.to_lowercase();
|
||||||
if valid_fields.contains(&normalized) {
|
if valid_fields.contains(&normalized) {
|
||||||
@@ -131,7 +124,7 @@ pub(super) fn validate_nats_server_config(server: &ServerAddr, config: &KVS, def
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let queue_dir = queue_dir_or_default(config, NATS_QUEUE_DIR, default_queue_dir);
|
let queue_dir = config.lookup(NATS_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string());
|
||||||
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
||||||
return Err(TargetError::Configuration("NATS queue directory must be an absolute path".to_string()));
|
return Err(TargetError::Configuration("NATS queue directory must be an absolute path".to_string()));
|
||||||
}
|
}
|
||||||
@@ -211,7 +204,9 @@ pub(super) fn validate_pulsar_broker_config(broker: &str, config: &KVS, default_
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let queue_dir = queue_dir_or_default(config, PULSAR_QUEUE_DIR, default_queue_dir);
|
let queue_dir = config
|
||||||
|
.lookup(PULSAR_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string());
|
||||||
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
||||||
return Err(TargetError::Configuration("Pulsar queue directory must be an absolute path".to_string()));
|
return Err(TargetError::Configuration("Pulsar queue directory must be an absolute path".to_string()));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::common::{
|
use super::common::{
|
||||||
parse_jetstream_ack_timeout_secs, parse_jetstream_enable, parse_target_bool, parse_url, queue_dir_or_default,
|
parse_jetstream_ack_timeout_secs, parse_jetstream_enable, parse_target_bool, parse_url, validate_nats_server_config,
|
||||||
validate_nats_server_config, validate_outbound_http_url, validate_pulsar_broker_config,
|
validate_outbound_http_url, validate_pulsar_broker_config,
|
||||||
};
|
};
|
||||||
use crate::error::TargetError;
|
use crate::error::TargetError;
|
||||||
use crate::target::{
|
use crate::target::{
|
||||||
@@ -118,7 +118,7 @@ pub fn build_amqp_args(config: &KVS, default_queue_dir: &str, target_type: Targe
|
|||||||
tls_ca: config.lookup(AMQP_TLS_CA).unwrap_or_default(),
|
tls_ca: config.lookup(AMQP_TLS_CA).unwrap_or_default(),
|
||||||
tls_client_cert: config.lookup(AMQP_TLS_CLIENT_CERT).unwrap_or_default(),
|
tls_client_cert: config.lookup(AMQP_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||||
tls_client_key: config.lookup(AMQP_TLS_CLIENT_KEY).unwrap_or_default(),
|
tls_client_key: config.lookup(AMQP_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||||
queue_dir: queue_dir_or_default(config, AMQP_QUEUE_DIR, default_queue_dir),
|
queue_dir: config.lookup(AMQP_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(AMQP_QUEUE_LIMIT)
|
.lookup(AMQP_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -146,7 +146,9 @@ pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: Ta
|
|||||||
enable: true,
|
enable: true,
|
||||||
endpoint: endpoint_url,
|
endpoint: endpoint_url,
|
||||||
auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
|
auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
|
||||||
queue_dir: queue_dir_or_default(config, WEBHOOK_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(WEBHOOK_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(WEBHOOK_QUEUE_LIMIT)
|
.lookup(WEBHOOK_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -178,7 +180,9 @@ pub fn validate_webhook_config(config: &KVS, default_queue_dir: &str) -> Result<
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let queue_dir = queue_dir_or_default(config, WEBHOOK_QUEUE_DIR, default_queue_dir);
|
let queue_dir = config
|
||||||
|
.lookup(WEBHOOK_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string());
|
||||||
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
||||||
return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
|
return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
|
||||||
}
|
}
|
||||||
@@ -230,7 +234,7 @@ pub fn build_mqtt_args(config: &KVS, default_queue_dir: &str, target_type: Targe
|
|||||||
config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
|
config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
|
||||||
config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
|
config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
|
||||||
)?,
|
)?,
|
||||||
queue_dir: queue_dir_or_default(config, MQTT_QUEUE_DIR, default_queue_dir),
|
queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(MQTT_QUEUE_LIMIT)
|
.lookup(MQTT_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -307,7 +311,7 @@ pub fn build_nats_args(config: &KVS, default_queue_dir: &str, target_type: Targe
|
|||||||
tls_client_cert: config.lookup(NATS_TLS_CLIENT_CERT).unwrap_or_default(),
|
tls_client_cert: config.lookup(NATS_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||||
tls_client_key: config.lookup(NATS_TLS_CLIENT_KEY).unwrap_or_default(),
|
tls_client_key: config.lookup(NATS_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||||
tls_required: parse_target_bool(config.lookup(NATS_TLS_REQUIRED).as_deref()).unwrap_or(false),
|
tls_required: parse_target_bool(config.lookup(NATS_TLS_REQUIRED).as_deref()).unwrap_or(false),
|
||||||
queue_dir: queue_dir_or_default(config, NATS_QUEUE_DIR, default_queue_dir),
|
queue_dir: config.lookup(NATS_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(NATS_QUEUE_LIMIT)
|
.lookup(NATS_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -349,7 +353,9 @@ pub fn build_pulsar_args(config: &KVS, default_queue_dir: &str, target_type: Tar
|
|||||||
tls_ca: config.lookup(PULSAR_TLS_CA).unwrap_or_default(),
|
tls_ca: config.lookup(PULSAR_TLS_CA).unwrap_or_default(),
|
||||||
tls_allow_insecure: parse_target_bool(config.lookup(PULSAR_TLS_ALLOW_INSECURE).as_deref()).unwrap_or(false),
|
tls_allow_insecure: parse_target_bool(config.lookup(PULSAR_TLS_ALLOW_INSECURE).as_deref()).unwrap_or(false),
|
||||||
tls_hostname_verification: parse_target_bool(config.lookup(PULSAR_TLS_HOSTNAME_VERIFICATION).as_deref()).unwrap_or(true),
|
tls_hostname_verification: parse_target_bool(config.lookup(PULSAR_TLS_HOSTNAME_VERIFICATION).as_deref()).unwrap_or(true),
|
||||||
queue_dir: queue_dir_or_default(config, PULSAR_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(PULSAR_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(PULSAR_QUEUE_LIMIT)
|
.lookup(PULSAR_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -399,7 +405,9 @@ pub fn build_redis_args(
|
|||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
.map(Duration::from_secs)
|
.map(Duration::from_secs)
|
||||||
.unwrap_or_else(|| Duration::from_secs(15)),
|
.unwrap_or_else(|| Duration::from_secs(15)),
|
||||||
queue_dir: queue_dir_or_default(config, REDIS_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(REDIS_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(REDIS_QUEUE_LIMIT)
|
.lookup(REDIS_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -455,7 +463,9 @@ pub fn build_postgres_args(config: &KVS, default_queue_dir: &str, target_type: T
|
|||||||
tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(),
|
tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(),
|
||||||
tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(),
|
tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||||
tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(),
|
tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||||
queue_dir: queue_dir_or_default(config, POSTGRES_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(POSTGRES_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(POSTGRES_QUEUE_LIMIT)
|
.lookup(POSTGRES_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -519,7 +529,9 @@ pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: Targ
|
|||||||
},
|
},
|
||||||
sasl_username,
|
sasl_username,
|
||||||
sasl_password,
|
sasl_password,
|
||||||
queue_dir: queue_dir_or_default(config, KAFKA_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(KAFKA_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(KAFKA_QUEUE_LIMIT)
|
.lookup(KAFKA_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
@@ -557,7 +569,9 @@ pub fn build_mysql_args(config: &KVS, default_queue_dir: &str, target_type: Targ
|
|||||||
tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(),
|
tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(),
|
||||||
tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(),
|
tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||||
tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(),
|
tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||||
queue_dir: queue_dir_or_default(config, MYSQL_QUEUE_DIR, default_queue_dir),
|
queue_dir: config
|
||||||
|
.lookup(MYSQL_QUEUE_DIR)
|
||||||
|
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||||
queue_limit: config
|
queue_limit: config
|
||||||
.lookup(MYSQL_QUEUE_LIMIT)
|
.lookup(MYSQL_QUEUE_LIMIT)
|
||||||
.and_then(|v| v.parse::<u64>().ok())
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
|
|||||||
@@ -15,13 +15,12 @@
|
|||||||
use crate::error::StoreError;
|
use crate::error::StoreError;
|
||||||
use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
|
use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, DEFAULT_TARGET_STORE_MAX_BYTES, ENV_TARGET_STORE_COMPRESS,
|
DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, DEFAULT_TARGET_STORE_MAX_BYTES, ENV_TARGET_STORE_COMPRESS,
|
||||||
ENV_TARGET_STORE_DIR, ENV_TARGET_STORE_MAX_BYTES, EVENT_DEFAULT_DIR, EnableState,
|
ENV_TARGET_STORE_MAX_BYTES, EnableState,
|
||||||
};
|
};
|
||||||
use serde::{Serialize, de::DeserializeOwned};
|
use serde::{Serialize, de::DeserializeOwned};
|
||||||
use snap::raw::{Decoder, Encoder};
|
use snap::raw::{Decoder, Encoder};
|
||||||
use std::{
|
use std::{
|
||||||
borrow::Cow,
|
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
fs::File,
|
fs::File,
|
||||||
io::Write,
|
io::Write,
|
||||||
@@ -156,46 +155,6 @@ fn queue_store_compression_enabled() -> bool {
|
|||||||
resolve_queue_store_compression_from_env_value(value.as_deref())
|
resolve_queue_store_compression_from_env_value(value.as_deref())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Map a configured target `queue_dir` to the directory this node opens.
|
|
||||||
///
|
|
||||||
/// `EVENT_DEFAULT_DIR` and `AUDIT_DEFAULT_DIR` are persisted cluster-wide as logical
|
|
||||||
/// defaults, so every node resolves them locally. `RUSTFS_TARGET_STORE_DIR` relocates
|
|
||||||
/// their root on any platform. Without it, Windows uses `%ProgramData%\RustFS`, because
|
|
||||||
/// the Unix literal has no drive prefix and is not an absolute path there. Every other
|
|
||||||
/// value, including an explicit operator path, is returned unchanged.
|
|
||||||
pub fn resolve_queue_dir(queue_dir: &str) -> Cow<'_, str> {
|
|
||||||
let store_root = std::env::var(ENV_TARGET_STORE_DIR).ok();
|
|
||||||
resolve_queue_dir_with(queue_dir, store_root.as_deref(), platform_queue_root().as_deref())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(windows)]
|
|
||||||
fn platform_queue_root() -> Option<PathBuf> {
|
|
||||||
std::env::var_os("ProgramData")
|
|
||||||
.filter(|root| !root.is_empty())
|
|
||||||
.map(|root| PathBuf::from(root).join("RustFS"))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(windows))]
|
|
||||||
fn platform_queue_root() -> Option<PathBuf> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn resolve_queue_dir_with<'a>(queue_dir: &'a str, store_root: Option<&str>, platform_root: Option<&Path>) -> Cow<'a, str> {
|
|
||||||
let leaf = match queue_dir {
|
|
||||||
EVENT_DEFAULT_DIR => "events",
|
|
||||||
AUDIT_DEFAULT_DIR => "audit",
|
|
||||||
_ => return Cow::Borrowed(queue_dir),
|
|
||||||
};
|
|
||||||
let root = match store_root.map(str::trim).filter(|root| !root.is_empty()) {
|
|
||||||
Some(root) => Path::new(root),
|
|
||||||
None => match platform_root {
|
|
||||||
Some(root) => root,
|
|
||||||
None => return Cow::Borrowed(queue_dir),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
Cow::Owned(root.join(leaf).to_string_lossy().into_owned())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Represents a key for an entry in the store
|
/// Represents a key for an entry in the store
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Key {
|
pub struct Key {
|
||||||
@@ -1406,56 +1365,6 @@ mod tests {
|
|||||||
thread,
|
thread,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn logical_default_queue_dirs_resolve_under_the_configured_root() {
|
|
||||||
let root = Path::new("/srv/rustfs-queues");
|
|
||||||
let platform = Path::new("/platform/RustFS");
|
|
||||||
assert_eq!(
|
|
||||||
resolve_queue_dir_with(EVENT_DEFAULT_DIR, Some("/srv/rustfs-queues"), Some(platform)),
|
|
||||||
root.join("events").to_string_lossy()
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
resolve_queue_dir_with(AUDIT_DEFAULT_DIR, Some(" /srv/rustfs-queues "), None),
|
|
||||||
root.join("audit").to_string_lossy()
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
resolve_queue_dir_with(EVENT_DEFAULT_DIR, Some(" "), Some(platform)),
|
|
||||||
platform.join("events").to_string_lossy(),
|
|
||||||
"a blank override must fall back to the platform root"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn explicit_and_unmapped_queue_dirs_are_preserved() {
|
|
||||||
let platform = Path::new("/platform/RustFS");
|
|
||||||
for explicit in [
|
|
||||||
"/var/lib/rustfs/events",
|
|
||||||
"/opt/rustfs/events/",
|
|
||||||
"/opt/rustfs",
|
|
||||||
"",
|
|
||||||
"relative/queue",
|
|
||||||
] {
|
|
||||||
assert_eq!(
|
|
||||||
resolve_queue_dir_with(explicit, Some("/srv/rustfs-queues"), Some(platform)),
|
|
||||||
explicit,
|
|
||||||
"only the exact logical defaults may be relocated"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(resolve_queue_dir_with(EVENT_DEFAULT_DIR, None, None), EVENT_DEFAULT_DIR);
|
|
||||||
assert_eq!(resolve_queue_dir_with(AUDIT_DEFAULT_DIR, None, None), AUDIT_DEFAULT_DIR);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(windows)]
|
|
||||||
#[test]
|
|
||||||
fn windows_logical_default_queue_dirs_are_absolute_without_an_override() {
|
|
||||||
let platform = platform_queue_root().expect("Windows defines ProgramData");
|
|
||||||
for default in [EVENT_DEFAULT_DIR, AUDIT_DEFAULT_DIR] {
|
|
||||||
let resolved = resolve_queue_dir_with(default, None, Some(&platform));
|
|
||||||
assert!(Path::new(resolved.as_ref()).is_absolute(), "{default} resolved to {resolved}");
|
|
||||||
assert!(resolved.contains("RustFS"), "{default} resolved to {resolved}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn temp_store_dir(name: &str) -> PathBuf {
|
fn temp_store_dir(name: &str) -> PathBuf {
|
||||||
std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
|
std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,18 +45,9 @@ In a distributed deployment, startup publishes `FullReady` only after that local
|
|||||||
- Liveness reports process availability and must not depend on storage, IAM,
|
- Liveness reports process availability and must not depend on storage, IAM,
|
||||||
lock quorum, or peer health.
|
lock quorum, or peer health.
|
||||||
- Node readiness (`/health/ready`, `/minio/health/ready`) reports whether this process can serve reads: storage read quorum, IAM, and shared-lock quorum. It is the Kubernetes Service membership signal. Losing write quorum, the pool-metadata writer, or exclusive-lock quorum does not remove a node that can still read. The public health handler still returns 503 until startup publishes `FullReady`, so a process that has never opened S3 admission does not join the Service. Runtime storage diagnostics do not change that startup gate.
|
- Node readiness (`/health/ready`, `/minio/health/ready`) reports whether this process can serve reads: storage read quorum, IAM, and shared-lock quorum. It is the Kubernetes Service membership signal. Losing write quorum, the pool-metadata writer, or exclusive-lock quorum does not remove a node that can still read. The public health handler still returns 503 until startup publishes `FullReady`, so a process that has never opened S3 admission does not join the Service. Runtime storage diagnostics do not change that startup gate.
|
||||||
- A blocked pool metadata writer degrades cluster-write readiness with
|
- Immutable pool metadata admission proofs share the local authority gate as readers. Publication, recovery, identity changes and operations requiring serialization retain exclusive ownership and their existing distributed namespace fences. A multi-phase operation releases its reader before acquiring the writer and revalidates under exclusive ownership; it never upgrades a held reader. The gate is fair, so a writer queued behind a reader that is waiting on namespace I/O also delays later admissions until that reader finishes. Every writer publishes the authority it leaves before releasing exclusive ownership. Health inspection reads that published authority without acquiring the gate, so neither a held nor a queued writer delays it. A writer's changes become visible when it releases the gate; a latch recorded by a transaction outcome after its writer released is shared with the published authority and visible immediately. Mutating paths still check authority under the gate.
|
||||||
`pool_meta_write_blocked`. It does not withdraw `/health/ready`, and the node
|
- A blocked pool metadata writer degrades cluster-write readiness with `pool_meta_write_blocked`. It does not withdraw `/health/ready`, and the node probe omits that reason from `degradedReasons`. Metadata authority inspection reads the state published by the last local writer and does not wait for the save gate, so a held or queued metadata writer does not degrade either probe; a block that writer installs appears when it releases the gate.
|
||||||
probe omits that reason from `degradedReasons`. Metadata save-gate inspection
|
- The authenticated cluster snapshot extends its existing node-local metadata gate inspection with safe reason, failure phase, and original block time. It changes no admission or recovery decision. Runtime readiness and gate status are separate observations.
|
||||||
is bounded to 100 ms; contention reports `pool_metadata_check_timeout` on the
|
|
||||||
cluster-write probe without installing a block. The node probe keeps the last
|
|
||||||
confirmed writable observation for `details.poolMetadata` during a transient
|
|
||||||
inspection timeout, while preserving a confirmed block and failing that
|
|
||||||
detail closed when no fresh observation exists.
|
|
||||||
- The authenticated cluster snapshot extends its existing node-local metadata
|
|
||||||
gate inspection with safe reason, failure phase, and original block time. It
|
|
||||||
distinguishes timeout from a block and changes no admission or recovery
|
|
||||||
decision. Runtime readiness and gate status are separate bounded observations.
|
|
||||||
- Cluster write readiness (`/minio/health/cluster`) requires storage write quorum, the pool-metadata write gate, and exclusive-lock quorum. Use it when a caller must know that new writes can commit. It is not the Service membership probe.
|
- Cluster write readiness (`/minio/health/cluster`) requires storage write quorum, the pool-metadata write gate, and exclusive-lock quorum. Use it when a caller must know that new writes can commit. It is not the Service membership probe.
|
||||||
- Cluster read readiness uses the storage read-quorum path and cluster-health timeout behavior. Its lock dependency still uses the per-set majority/write-lock health check. Actual shared namespace locks require `n - n/2`, so the cluster read probe is conservative: a four-client set can still admit reads with two clients while that probe returns 503. Node `/health/ready` uses the shared-lock threshold instead, because that is what object reads acquire.
|
- Cluster read readiness uses the storage read-quorum path and cluster-health timeout behavior. Its lock dependency still uses the per-set majority/write-lock health check. Actual shared namespace locks require `n - n/2`, so the cluster read probe is conservative: a four-client set can still admit reads with two clients while that probe returns 503. Node `/health/ready` uses the shared-lock threshold instead, because that is what object reads acquire.
|
||||||
- `HEAD` health probes keep header/status semantics and do not require response
|
- `HEAD` health probes keep header/status semantics and do not require response
|
||||||
@@ -91,7 +82,7 @@ For a healthy IAM and metadata writer in a four-node, one-drive-per-node EC 2+2
|
|||||||
|
|
||||||
With two of four nodes up, GET, HEAD, and List of a bucket that this process has not yet cached still succeed. PUT and other mutations fail because write quorum and exclusive locks are gone.
|
With two of four nodes up, GET, HEAD, and List of a bucket that this process has not yet cached still succeed. PUT and other mutations fail because write quorum and exclusive locks are gone.
|
||||||
|
|
||||||
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection retains its own 100 ms budget; a timeout is counted by `rustfs_pool_metadata_check_timeouts_total` and uses the last confirmed node-local gate state only within the same cache TTL. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection reads the authority published by the last local writer and does not wait for the save gate. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
||||||
|
|
||||||
The new fields are additive. Their absence in an older response is not evidence of storage quorum. Minimal responses still contain only the existing top-level fields, liveness remains dependency-independent, and HEAD responses remain bodyless.
|
The new fields are additive. Their absence in an older response is not evidence of storage quorum. Minimal responses still contain only the existing top-level fields, liveness remains dependency-independent, and HEAD responses remain bodyless.
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
|||||||
| `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias; smoke profile defaults to signing name `s3tables`. |
|
| `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias; smoke profile defaults to signing name `s3tables`. |
|
||||||
| S3 object data plane | Supported | Data, metadata, manifest, and delete files are ordinary S3 objects with table-aware policy checks on warehouse paths. |
|
| S3 object data plane | Supported | Data, metadata, manifest, and delete files are ordinary S3 objects with table-aware policy checks on warehouse paths. |
|
||||||
| S3 table-bucket listings | Supported | `ListObjects`, `ListObjectsV2`, `ListObjectVersions`, metadata-list extensions, and multipart-upload listings omit warehouse objects unless the caller has `admin:GetTableMetadata` for that table, and always omit internal `.rustfs-table` keys. Protected cursors are encrypted with a key derived from the cluster root credentials, so rotating those credentials invalidates cursors already issued. |
|
| S3 table-bucket listings | Supported | `ListObjects`, `ListObjectsV2`, `ListObjectVersions`, metadata-list extensions, and multipart-upload listings omit warehouse objects unless the caller has `admin:GetTableMetadata` for that table, and always omit internal `.rustfs-table` keys. Protected cursors are encrypted with a key derived from the cluster root credentials, so rotating those credentials invalidates cursors already issued. |
|
||||||
| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index by listing the catalog prefix. Only table entries count toward the 4,096-object limit, so maintenance reports and other per-table objects cannot exhaust it; the listing itself is bounded to 64 pages plus one page per table entry already listed, and after each page the cursor skips the rest of the maintenance subtree the page ended in, so a report flood under one table costs a bounded number of pages. Larger catalogs fail closed with a retryable service error and require operator backfill. An optional table-entry directory (see Maintenance Matrix) answers these lookups without the prefix listing. |
|
| Object-backed warehouse-index recovery | Supported with bounded request fallback | Data-plane requests repair or verify a missing warehouse-prefix index using authoritative table-entry pages. The scan selects `table-entry.json` below identifier-like directory components at any depth; namespace components named `tables` or `maintenance` remain valid. Noncandidate report names are filtered in directory batches before inventory retention and metadata reads, so report history does not consume ownership pages. Selected entries retain GET-equivalent quorum and corruption checks. More than 4,096 table entries fails closed with a retryable service error and requires operator backfill. An optional table-entry directory (see Maintenance Matrix) answers these lookups without the prefix listing. |
|
||||||
| Table bucket enablement | Supported | A regular bucket is enabled for catalog use and addressed as the REST catalog warehouse. |
|
| Table bucket enablement | Supported | A regular bucket is enabled for catalog use and addressed as the REST catalog warehouse. |
|
||||||
| Catalog-vended table credentials | Automated when enabled | Disabled by default. LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. |
|
| Catalog-vended table credentials | Automated when enabled | Disabled by default. LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. |
|
||||||
| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and warehouse ARN shape for migration docs. API parity not claimed. |
|
| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and warehouse ARN shape for migration docs. API parity not claimed. |
|
||||||
@@ -132,6 +132,8 @@ Object-backed commit publication identifies every catalog object it validated by
|
|||||||
|
|
||||||
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and restores require disabling the directory first. Object catalog backups exclude the derived directory state and candidates. Re-enable the directory after a restore to rebuild it from the restored entries.
|
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and restores require disabling the directory first. Object catalog backups exclude the derived directory state and candidates. Re-enable the directory after a restore to rebuild it from the restored entries.
|
||||||
|
|
||||||
|
Fallback ownership scans, including the check before metadata cleanup, use the optional `identifier_tree_leaf` walk selector. An absent selector preserves ordinary listing behavior. Older peers ignore the additive field; the coordinator filters their noncandidate rows before metadata resolution and retains read-quorum checks for every selected entry. Old peers may do more I/O or fail on unrelated corrupt metadata, so a mixed-version scan can refuse work until those peers are upgraded. Such failures never establish owner absence. This selector changes no catalog or on-disk format and requires no fleet-confirmation flag.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
| Capability | Status | Claim |
|
| Capability | Status | Claim |
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ An unreadable replica, lost fence, or conditional-write conflict leaves the orig
|
|||||||
|
|
||||||
- The first block emits `decommission_state` with `state=pool_metadata_blocked`, `reason`, `phase`, and `blocked_since`. A change in recovery failure classification emits `state=pool_metadata_recovery_pending`; successful recovery emits `state=pool_metadata_recovered` with the original timestamp.
|
- The first block emits `decommission_state` with `state=pool_metadata_blocked`, `reason`, `phase`, and `blocked_since`. A change in recovery failure classification emits `state=pool_metadata_recovery_pending`; successful recovery emits `state=pool_metadata_recovered` with the original timestamp.
|
||||||
- `rustfs_pool_metadata_blocks_total{reason}` and `rustfs_pool_metadata_recoveries_total` count block and recovery transitions. The original cause and phase remain attached to local typed errors; storage/RPC error numbers and on-disk formats are unchanged.
|
- `rustfs_pool_metadata_blocks_total{reason}` and `rustfs_pool_metadata_recoveries_total` count block and recovery transitions. The original cause and phase remain attached to local typed errors; storage/RPC error numbers and on-disk formats are unchanged.
|
||||||
- Node and cluster-write readiness include `pool_meta_write_blocked`. Waiting for the metadata save mutex is bounded to 100 ms and reports `pool_metadata_check_timeout`, not a persistent block. Cluster probes retain their existing cache and overall timeout behavior. Liveness and cluster-read quorum checks are unchanged. The authenticated node-local status and safe error fields are described in [S3 write failure diagnostics](s3-write-failure-diagnostics.md).
|
- Cluster-write readiness reports `pool_meta_write_blocked`; node readiness reports the writer only through `details.poolMetadata`. Readiness reads the authority published by the last local metadata writer instead of waiting for the save gate, so an active or queued metadata transaction does not degrade readiness by itself. A block installed by a writer appears when that writer releases the gate. Cluster probes retain their existing cache and overall timeout behavior. Liveness and cluster-read quorum checks are unchanged. The authenticated node-local status and safe error fields are described in [S3 write failure diagnostics](s3-write-failure-diagnostics.md).
|
||||||
|
|
||||||
Pool metadata CAS explicitly selects `TailDrainedWithOwnerRepair`: it waits for the complete rename result and returns repair responsibility to its existing deployment and transaction owner when write quorum commits but a replica fails or signatures diverge. Cache invalidation and the committed cleanup handoff precede that error. Ordinary system metadata retains quorum-success semantics under `Quorum` and `TailDrained`; draining a tail does not require every disk to succeed. A repair-required completion does not enter rollback cleanup or wait for another temporary-data deletion. The committed bytes are not rolled back. Generic MRF and heal-channel requests cannot supply the deployment and transaction authority needed to repair system metadata; pool CAS retains the uncertain outcome and its existing recovery fence. Ordinary object repair keeps its bucket-incarnation fencing.
|
Pool metadata CAS explicitly selects `TailDrainedWithOwnerRepair`: it waits for the complete rename result and returns repair responsibility to its existing deployment and transaction owner when write quorum commits but a replica fails or signatures diverge. Cache invalidation and the committed cleanup handoff precede that error. Ordinary system metadata retains quorum-success semantics under `Quorum` and `TailDrained`; draining a tail does not require every disk to succeed. A repair-required completion does not enter rollback cleanup or wait for another temporary-data deletion. The committed bytes are not rolled back. Generic MRF and heal-channel requests cannot supply the deployment and transaction authority needed to repair system metadata; pool CAS retains the uncertain outcome and its existing recovery fence. Ordinary object repair keeps its bucket-incarnation fencing.
|
||||||
|
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ When the whole cluster (or several nodes) went down and nodes are brought back o
|
|||||||
|
|
||||||
2. Logs say what the node waits for. The IAM recovery loop retries with backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable `hint` field (for example, "storage read quorum not met yet; waiting for enough cluster nodes/disks to come online"). After repeated failures the level escalates from WARN to ERROR; this still does not kill the process.
|
2. Logs say what the node waits for. The IAM recovery loop retries with backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable `hint` field (for example, "storage read quorum not met yet; waiting for enough cluster nodes/disks to come online"). After repeated failures the level escalates from WARN to ERROR; this still does not kill the process.
|
||||||
3. Recovery is automatic. Once storage read quorum is available, pending nodes can finish IAM bootstrap on the next retry. `/health/ready` returns `200` when startup has published `FullReady` and the node can still serve reads (storage read quorum, IAM, and shared-lock quorum). Write quorum is reported separately and is required by `/minio/health/cluster`, not by Service membership. Restarting pending nodes does not speed this up.
|
3. Recovery is automatic. Once storage read quorum is available, pending nodes can finish IAM bootstrap on the next retry. `/health/ready` returns `200` when startup has published `FullReady` and the node can still serve reads (storage read quorum, IAM, and shared-lock quorum). Write quorum is reported separately and is required by `/minio/health/cluster`, not by Service membership. Restarting pending nodes does not speed this up.
|
||||||
4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) separate `storage` / `poolMetadata` / `iam` / `lock` readiness. `storage.ready` follows read quorum (`readinessScope` is `read_quorum`); `storage.readQuorum` and `storage.writeQuorum` show the two observations. `poolMetadata.ready` is the metadata writer and does not by itself withdraw `/health/ready`. `degradedReasons` on this probe names read-path causes such as `storage_quorum_unavailable` and `storage_and_lock_unavailable`. `pool_meta_write_blocked` and `pool_metadata_check_timeout` stay on `/minio/health/cluster`. See the [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) for probe scopes and sampling limits:
|
4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) separate `storage` / `poolMetadata` / `iam` / `lock` readiness. `storage.ready` follows read quorum (`readinessScope` is `read_quorum`); `storage.readQuorum` and `storage.writeQuorum` show the two observations. `poolMetadata.ready` is the metadata writer and does not by itself withdraw `/health/ready`. `degradedReasons` on this probe names read-path causes such as `storage_quorum_unavailable` and `storage_and_lock_unavailable`. `pool_meta_write_blocked` stays on `/minio/health/cluster`. See the [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) for probe scopes and sampling limits:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -s http://<node>:9000/health/ready | jq
|
curl -s http://<node>:9000/health/ready | jq
|
||||||
|
|||||||
@@ -45,15 +45,14 @@ The probe round timeout is configured independently; see [Admin peer probe timeo
|
|||||||
|
|
||||||
## Correlate bounded diagnostics
|
## Correlate bounded diagnostics
|
||||||
|
|
||||||
Normal operation does not require success logs at WARN. Request counters remain available with WARN logging, while runtime readiness diagnostics distinguish `pool_meta_write_blocked`, `pool_metadata_check_timeout`, and insufficient storage quorum. Do not clear a metadata write fence merely to make readiness green.
|
Normal operation does not require success logs at WARN. Request counters remain available with WARN logging, while runtime readiness diagnostics distinguish `pool_meta_write_blocked` from insufficient storage quorum. Do not clear a metadata write fence merely to make readiness green.
|
||||||
|
|
||||||
Query the authenticated `/rustfs/admin/v4/cluster/snapshot` endpoint on the affected node. `snapshot.pool_meta_write_gate` describes that node's metadata writer, not a fleet-wide aggregate. Its existing booleans are preserved; `state` and optional block details extend the same bounded, read-only gate inspection. This starts no recovery, disk reads, or additional RPCs. Runtime readiness and the gate are inspected separately, so a whole cluster snapshot is not atomic across sections.
|
Query the authenticated `/rustfs/admin/v4/cluster/snapshot` endpoint on the affected node. `snapshot.pool_meta_write_gate` describes that node's metadata writer, not a fleet-wide aggregate. Its existing booleans are preserved; `state` and optional block details extend the same read-only inspection, which reads the authority published by the node's last metadata writer without waiting for the save gate. A block that an active writer installs appears when that writer releases the gate. Nodes running earlier releases can also report `check_timeout`, the `pool_metadata_check_timeout` readiness reason, and `rustfs_pool_metadata_check_timeouts_total` when their 100 ms gate inspection expired; current nodes emit none of them. This starts no recovery, disk reads, or additional RPCs. Runtime readiness and the gate are inspected separately, so a whole cluster snapshot is not atomic across sections.
|
||||||
|
|
||||||
| `state` | Meaning |
|
| `state` | Meaning |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `writable` | No metadata write block was observed. Other dependencies may still make the node unready. |
|
| `writable` | No metadata write block was observed. Other dependencies may still make the node unready. |
|
||||||
| `blocked` | A metadata write block was observed; `reason`, `phase`, and `sinceUnixSecs` describe its typed failure context. |
|
| `blocked` | A metadata write block was observed; `reason`, `phase`, and `sinceUnixSecs` describe its typed failure context. |
|
||||||
| `check_timeout` | The existing 100 ms inspection budget expired. Readiness remains false, but a write block is not asserted. |
|
|
||||||
| `unavailable` | The object store or a recognized metadata observation was unavailable; no block details are invented. |
|
| `unavailable` | The object store or a recognized metadata observation was unavailable; no block details are invented. |
|
||||||
|
|
||||||
For `blocked`, `reason` reuses the metadata failure classification, `phase` identifies the operation stage that caused the block (not live recovery-worker progress), and `sinceUnixSecs` is the original block time in Unix seconds. Polling does not reset that time. Recovery replaces the observation with `writable` and removes the previous block details. An unavailable store reports `writesReady=false` with `state="unavailable"`. Older responses may omit `state` and block details; missing fields alone do not prove a healthy writer. Operation text, raw replica errors, disk paths, and credentials are excluded. See [Pool metadata recovery](pool-metadata-recovery.md) for recovery and escalation boundaries.
|
For `blocked`, `reason` reuses the metadata failure classification, `phase` identifies the operation stage that caused the block (not live recovery-worker progress), and `sinceUnixSecs` is the original block time in Unix seconds. Polling does not reset that time. Recovery replaces the observation with `writable` and removes the previous block details. An unavailable store reports `writesReady=false` with `state="unavailable"`. Older responses may omit `state` and block details; missing fields alone do not prove a healthy writer. Operation text, raw replica errors, disk paths, and credentials are excluded. See [Pool metadata recovery](pool-metadata-recovery.md) for recovery and escalation boundaries.
|
||||||
|
|||||||
@@ -1,22 +1,4 @@
|
|||||||
# Target queue storage
|
# Target queue storage limits
|
||||||
|
|
||||||
## Queue directory location
|
|
||||||
|
|
||||||
A target's `queue_dir` is stored in the cluster-wide target configuration. The built-in defaults `/opt/rustfs/events` and `/opt/rustfs/audit` are logical defaults: each node resolves them to a local directory when it validates or opens a queue. The stock notify and audit default tables both persist `/opt/rustfs/events`; `/opt/rustfs/audit` applies only to an audit target whose configuration has no `queue_dir` key. Any other `queue_dir`, including an explicit operator path, is used as written and must be absolute.
|
|
||||||
|
|
||||||
| Node setting | `/opt/rustfs/events` resolves to | `/opt/rustfs/audit` resolves to |
|
|
||||||
|---|---|---|
|
|
||||||
| `RUSTFS_TARGET_STORE_DIR=<root>` (any platform) | `<root>/events` | `<root>/audit` |
|
|
||||||
| Windows, variable unset | `%ProgramData%\RustFS\events` | `%ProgramData%\RustFS\audit` |
|
|
||||||
| Linux/macOS, variable unset | `/opt/rustfs/events` | `/opt/rustfs/audit` |
|
|
||||||
|
|
||||||
`RUSTFS_TARGET_STORE_DIR` must be an absolute path; a relative value makes target validation fail with the queue-directory error. The queue store creates missing directories when it opens, so the resolved directory only needs to be creatable by the RustFS process. An admin request that names a default path explicitly is checked at its resolved location and, as for any explicit `queue_dir`, that directory must already exist.
|
|
||||||
|
|
||||||
The official container image links `/opt/rustfs/events` and `/opt/rustfs/audit` into `/data/.rustfs`, so its queues survive container recreation without this variable. Windows previously rejected targets that kept the default because the Unix literal is not an absolute Windows path; those targets now validate and queue under `%ProgramData%\RustFS`.
|
|
||||||
|
|
||||||
Setting or changing `RUSTFS_TARGET_STORE_DIR` on an existing node moves where its default queues are opened. Entries still pending in the previous directory are not migrated: let them drain first, or move the directory contents while RustFS is stopped. Rolling back to a version without this variable returns default queues to the previous location.
|
|
||||||
|
|
||||||
## Queue size limits
|
|
||||||
|
|
||||||
Disk queues for notification and audit targets enforce both their configured entry limit and a shared byte limit for each queue directory. `RUSTFS_TARGET_STORE_MAX_BYTES` sets that byte limit; the default is 1 GiB. The value must be a positive integer in bytes. Missing, zero, invalid, and overflowing values use the default.
|
Disk queues for notification and audit targets enforce both their configured entry limit and a shared byte limit for each queue directory. `RUSTFS_TARGET_STORE_MAX_BYTES` sets that byte limit; the default is 1 GiB. The value must be a positive integer in bytes. Missing, zero, invalid, and overflowing values use the default.
|
||||||
|
|
||||||
|
|||||||
@@ -157,7 +157,6 @@ behavior.
|
|||||||
| config.rustfs.obs_environment | string | `"development"` | |
|
| config.rustfs.obs_environment | string | `"development"` | |
|
||||||
| config.rustfs.obs_log_directory | string | `"/logs"` | Log directory inside the RustFS container. Set to `""` to disable log PVCs and mounts. |
|
| config.rustfs.obs_log_directory | string | `"/logs"` | Log directory inside the RustFS container. Set to `""` to disable log PVCs and mounts. |
|
||||||
| config.rustfs.region | string | `"us-east-1"` | |
|
| config.rustfs.region | string | `"us-east-1"` | |
|
||||||
| config.rustfs.target_store_dir | string | `""` | Absolute root for notify and audit target queues that keep the default `queue_dir`, set as `RUSTFS_TARGET_STORE_DIR`; queues move to `<root>/events` and `<root>/audit`. Empty keeps the image layout, where `/opt/rustfs/events` and `/opt/rustfs/audit` link into `/data/.rustfs`. |
|
|
||||||
| config.rustfs.volumes | string | `""` | Explicit distributed volume topology. When empty, the chart generates the topology and normally injects `RUSTFS_LOCAL_ENDPOINT_HOST`; custom topologies must configure local endpoint identity explicitly when needed. |
|
| config.rustfs.volumes | string | `""` | Explicit distributed volume topology. When empty, the chart generates the topology and normally injects `RUSTFS_LOCAL_ENDPOINT_HOST`; custom topologies must configure local endpoint identity explicitly when needed. |
|
||||||
| config.rustfs.log_rotation.size | int | `"100"` | Default log rotation size mb for rustfs. |
|
| config.rustfs.log_rotation.size | int | `"100"` | Default log rotation size mb for rustfs. |
|
||||||
| config.rustfs.log_rotation.time | string | `"hour"` | Default log rotation time for rustfs. |
|
| config.rustfs.log_rotation.time | string | `"hour"` | Default log rotation time for rustfs. |
|
||||||
|
|||||||
@@ -21,9 +21,6 @@ data:
|
|||||||
{{- if .region }}
|
{{- if .region }}
|
||||||
RUSTFS_REGION: {{ .region | quote }}
|
RUSTFS_REGION: {{ .region | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- if .target_store_dir }}
|
|
||||||
RUSTFS_TARGET_STORE_DIR: {{ .target_store_dir | quote }}
|
|
||||||
{{- end }}
|
|
||||||
{{- if .domains }}
|
{{- if .domains }}
|
||||||
RUSTFS_SERVER_DOMAINS: {{ include "rustfs.serverDomains" $ | quote }}
|
RUSTFS_SERVER_DOMAINS: {{ include "rustfs.serverDomains" $ | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|||||||
@@ -143,9 +143,6 @@ config:
|
|||||||
console_enable: "true"
|
console_enable: "true"
|
||||||
console_address: ":9001"
|
console_address: ":9001"
|
||||||
region: "us-east-1"
|
region: "us-east-1"
|
||||||
# Root for notify/audit target queues that keep the default queue_dir; queues use
|
|
||||||
# <root>/events and <root>/audit. Empty keeps the image layout (/opt/rustfs -> /data/.rustfs).
|
|
||||||
target_store_dir: ""
|
|
||||||
# Optionally enable support for virtual-hosted-style requests.
|
# Optionally enable support for virtual-hosted-style requests.
|
||||||
# See more information: https://docs.rustfs.com/integration/virtual.html
|
# See more information: https://docs.rustfs.com/integration/virtual.html
|
||||||
domains: "" # e.g. "example.com"
|
domains: "" # e.g. "example.com"
|
||||||
|
|||||||
@@ -211,9 +211,7 @@ pub(crate) struct ClusterPoolMetaWriteGateView {
|
|||||||
|
|
||||||
impl From<ClusterPoolMetaWriteGateSnapshot> for ClusterPoolMetaWriteGateView {
|
impl From<ClusterPoolMetaWriteGateSnapshot> for ClusterPoolMetaWriteGateView {
|
||||||
fn from(snapshot: ClusterPoolMetaWriteGateSnapshot) -> Self {
|
fn from(snapshot: ClusterPoolMetaWriteGateSnapshot) -> Self {
|
||||||
let state = if snapshot.check_timed_out {
|
let state = if snapshot.writes_ready {
|
||||||
"check_timeout"
|
|
||||||
} else if snapshot.writes_ready {
|
|
||||||
"writable"
|
"writable"
|
||||||
} else if snapshot.write_blocked || snapshot.transaction_aborted {
|
} else if snapshot.write_blocked || snapshot.transaction_aborted {
|
||||||
"blocked"
|
"blocked"
|
||||||
@@ -706,7 +704,6 @@ fn summarize_storage_readiness(snapshot: &ClusterReadOnlySnapshot) -> Capability
|
|||||||
.filter_map(|reason| match reason {
|
.filter_map(|reason| match reason {
|
||||||
ReadinessDegradedReason::StorageQuorumUnavailable
|
ReadinessDegradedReason::StorageQuorumUnavailable
|
||||||
| ReadinessDegradedReason::PoolMetaWriteBlocked
|
| ReadinessDegradedReason::PoolMetaWriteBlocked
|
||||||
| ReadinessDegradedReason::PoolMetadataCheckTimeout
|
|
||||||
| ReadinessDegradedReason::StorageAndIamUnavailable
|
| ReadinessDegradedReason::StorageAndIamUnavailable
|
||||||
| ReadinessDegradedReason::StorageAndLockUnavailable
|
| ReadinessDegradedReason::StorageAndLockUnavailable
|
||||||
| ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()),
|
| ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()),
|
||||||
@@ -1120,14 +1117,6 @@ mod tests {
|
|||||||
|
|
||||||
for (snapshot, state) in [
|
for (snapshot, state) in [
|
||||||
(ClusterPoolMetaWriteGateSnapshot::default(), "writable"),
|
(ClusterPoolMetaWriteGateSnapshot::default(), "writable"),
|
||||||
(
|
|
||||||
ClusterPoolMetaWriteGateSnapshot {
|
|
||||||
writes_ready: false,
|
|
||||||
check_timed_out: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
"check_timeout",
|
|
||||||
),
|
|
||||||
(
|
(
|
||||||
ClusterPoolMetaWriteGateSnapshot {
|
ClusterPoolMetaWriteGateSnapshot {
|
||||||
writes_ready: false,
|
writes_ready: false,
|
||||||
|
|||||||
@@ -560,8 +560,6 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
|
pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
|
||||||
let queue_dir = rustfs_targets::store::resolve_queue_dir(queue_dir);
|
|
||||||
let queue_dir = queue_dir.as_ref();
|
|
||||||
if !queue_dir.is_empty() {
|
if !queue_dir.is_empty() {
|
||||||
if !Path::new(queue_dir).is_absolute() {
|
if !Path::new(queue_dir).is_absolute() {
|
||||||
return Err(s3_error!(InvalidArgument, "queue_dir must be an absolute path"));
|
return Err(s3_error!(InvalidArgument, "queue_dir must be an absolute path"));
|
||||||
@@ -1076,58 +1074,3 @@ mod instance_runtime_status_tests {
|
|||||||
assert!(statuses.contains_key(&("PRIMARY".into(), "custom-two".into())));
|
assert!(statuses.contains_key(&("PRIMARY".into(), "custom-two".into())));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod queue_dir_resolution_tests {
|
|
||||||
use super::validate_queue_dir;
|
|
||||||
use rustfs_config::server_config::KVS;
|
|
||||||
use rustfs_config::{AUDIT_DEFAULT_DIR, ENV_TARGET_STORE_DIR, EVENT_DEFAULT_DIR, REDIS_QUEUE_DIR, REDIS_URL};
|
|
||||||
use rustfs_targets::{config::build_redis_args, target::TargetType};
|
|
||||||
use serial_test::serial;
|
|
||||||
|
|
||||||
fn redis_config() -> KVS {
|
|
||||||
let mut config = KVS::new();
|
|
||||||
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/".to_string());
|
|
||||||
config
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn default_queue_dirs_follow_the_target_store_root() {
|
|
||||||
let root = tempfile::tempdir().expect("queue store root");
|
|
||||||
let root_text = root.path().to_str().expect("UTF-8 queue store root").to_owned();
|
|
||||||
temp_env::async_with_vars([(ENV_TARGET_STORE_DIR, Some(root_text.as_str()))], async {
|
|
||||||
let notify = build_redis_args(&redis_config(), EVENT_DEFAULT_DIR, "events", TargetType::NotifyEvent)
|
|
||||||
.expect("default notify queue");
|
|
||||||
assert_eq!(notify.queue_dir, root.path().join("events").to_string_lossy());
|
|
||||||
let audit =
|
|
||||||
build_redis_args(&redis_config(), AUDIT_DEFAULT_DIR, "audit", TargetType::AuditLog).expect("default audit queue");
|
|
||||||
assert_eq!(audit.queue_dir, root.path().join("audit").to_string_lossy());
|
|
||||||
|
|
||||||
let explicit = root.path().join("operator-queue").to_string_lossy().into_owned();
|
|
||||||
let mut config = redis_config();
|
|
||||||
config.insert(REDIS_QUEUE_DIR.to_string(), explicit.clone());
|
|
||||||
let args = build_redis_args(&config, EVENT_DEFAULT_DIR, "events", TargetType::NotifyEvent).expect("explicit queue");
|
|
||||||
assert_eq!(args.queue_dir, explicit, "an operator path must not be relocated");
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn explicit_logical_default_is_validated_at_its_resolved_directory() {
|
|
||||||
let root = tempfile::tempdir().expect("queue store root");
|
|
||||||
let root_text = root.path().to_str().expect("UTF-8 queue store root").to_owned();
|
|
||||||
temp_env::async_with_vars([(ENV_TARGET_STORE_DIR, Some(root_text.as_str()))], async {
|
|
||||||
let missing = validate_queue_dir(EVENT_DEFAULT_DIR)
|
|
||||||
.await
|
|
||||||
.expect_err("the resolved queue directory does not exist yet");
|
|
||||||
assert!(missing.message().is_some_and(|message| message.contains("does not exist")));
|
|
||||||
std::fs::create_dir(root.path().join("events")).expect("resolved queue directory");
|
|
||||||
validate_queue_dir(EVENT_DEFAULT_DIR)
|
|
||||||
.await
|
|
||||||
.expect("the logical default must be checked where this node stores it");
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -108,7 +108,6 @@ pub struct ClusterListingDiagnosticsSnapshot {
|
|||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct ClusterPoolMetaWriteGateSnapshot {
|
pub struct ClusterPoolMetaWriteGateSnapshot {
|
||||||
pub writes_ready: bool,
|
pub writes_ready: bool,
|
||||||
pub check_timed_out: bool,
|
|
||||||
pub write_blocked: bool,
|
pub write_blocked: bool,
|
||||||
pub transaction_aborted: bool,
|
pub transaction_aborted: bool,
|
||||||
pub pool_meta_absent: bool,
|
pub pool_meta_absent: bool,
|
||||||
@@ -124,7 +123,6 @@ impl Default for ClusterPoolMetaWriteGateSnapshot {
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
writes_ready: true,
|
writes_ready: true,
|
||||||
check_timed_out: false,
|
|
||||||
write_blocked: false,
|
write_blocked: false,
|
||||||
transaction_aborted: false,
|
transaction_aborted: false,
|
||||||
pool_meta_absent: false,
|
pool_meta_absent: false,
|
||||||
@@ -230,7 +228,6 @@ async fn current_pool_meta_write_gate_snapshot() -> ClusterPoolMetaWriteGateSnap
|
|||||||
let status = store.pool_meta_write_gate_status().await;
|
let status = store.pool_meta_write_gate_status().await;
|
||||||
ClusterPoolMetaWriteGateSnapshot {
|
ClusterPoolMetaWriteGateSnapshot {
|
||||||
writes_ready: status.writes_ready,
|
writes_ready: status.writes_ready,
|
||||||
check_timed_out: status.check_timed_out,
|
|
||||||
write_blocked: status.write_blocked,
|
write_blocked: status.write_blocked,
|
||||||
transaction_aborted: status.transaction_aborted,
|
transaction_aborted: status.transaction_aborted,
|
||||||
pool_meta_absent: status.pool_meta_absent,
|
pool_meta_absent: status.pool_meta_absent,
|
||||||
|
|||||||
+65
-3
@@ -28,6 +28,19 @@ const MAX_VERSIONS_EXCEEDED_MESSAGE: &str = "You've exceeded the limit on the nu
|
|||||||
const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";
|
const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";
|
||||||
const SLOW_DOWN_READ_MESSAGE: &str = "Resource requested is unreadable, please reduce your request rate";
|
const SLOW_DOWN_READ_MESSAGE: &str = "Resource requested is unreadable, please reduce your request rate";
|
||||||
const NAMESPACE_LOCK_TIMEOUT_CAUSE: &str = "namespace_lock_timeout";
|
const NAMESPACE_LOCK_TIMEOUT_CAUSE: &str = "namespace_lock_timeout";
|
||||||
|
const RPC_ADMISSION_METADATA_KEY: &str = "x-rustfs-rpc-admission";
|
||||||
|
const RPC_REPLAY_ADMISSION_METADATA_VALUE: &str = "replay-cache";
|
||||||
|
|
||||||
|
/// Preserve temporary replay admission refusal through the RPC and S3 boundaries.
|
||||||
|
pub(crate) fn rpc_replay_admission_exhausted_status() -> tonic::Status {
|
||||||
|
let mut status = tonic::Status::resource_exhausted("RPC replay admission exhausted");
|
||||||
|
status.metadata_mut().insert(
|
||||||
|
RPC_ADMISSION_METADATA_KEY,
|
||||||
|
tonic::metadata::MetadataValue::from_static(RPC_REPLAY_ADMISSION_METADATA_VALUE),
|
||||||
|
);
|
||||||
|
status
|
||||||
|
}
|
||||||
|
|
||||||
/// Public message for `StorageError::UnsupportedDisk` on a request path. The
|
/// Public message for `StorageError::UnsupportedDisk` on a request path. The
|
||||||
/// only request-time producer is the delete-transaction protocol when a
|
/// only request-time producer is the delete-transaction protocol when a
|
||||||
/// participant answers Unimplemented (a peer still on a release without the
|
/// participant answers Unimplemented (a peer still on a release without the
|
||||||
@@ -588,9 +601,14 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
|
|||||||
return Some(S3ErrorCode::BadDigest);
|
return Some(S3ErrorCode::BadDigest);
|
||||||
}
|
}
|
||||||
|
|
||||||
rpc_unavailable |= error
|
rpc_unavailable |= error.downcast_ref::<tonic::Status>().is_some_and(|status| {
|
||||||
.downcast_ref::<tonic::Status>()
|
status.code() == tonic::Code::Unavailable
|
||||||
.is_some_and(|status| status.code() == tonic::Code::Unavailable);
|
|| (status.code() == tonic::Code::ResourceExhausted
|
||||||
|
&& status
|
||||||
|
.metadata()
|
||||||
|
.get(RPC_ADMISSION_METADATA_KEY)
|
||||||
|
.is_some_and(|value| value.as_encoded_bytes() == RPC_REPLAY_ADMISSION_METADATA_VALUE.as_bytes()))
|
||||||
|
});
|
||||||
upload_limit_exceeded |= error.is::<UploadLimitExceeded>();
|
upload_limit_exceeded |= error.is::<UploadLimitExceeded>();
|
||||||
server_source_failed |= error.is::<ServerSideSourceReadError>();
|
server_source_failed |= error.is::<ServerSideSourceReadError>();
|
||||||
client_body_timed_out |= error.is::<ClientBodyReadTimeout>();
|
client_body_timed_out |= error.is::<ClientBodyReadTimeout>();
|
||||||
@@ -1742,6 +1760,50 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_admission_refusal_preserves_retryable_status_across_io_boundaries() {
|
||||||
|
let status = rpc_replay_admission_exhausted_status;
|
||||||
|
let original = StorageError::from(status());
|
||||||
|
let cloned = original.clone();
|
||||||
|
let wire = status().into_http::<()>();
|
||||||
|
let received = tonic::Status::from_header_map(wire.headers()).expect("replay admission status on the wire");
|
||||||
|
for error in [
|
||||||
|
ApiError::from(original),
|
||||||
|
ApiError::from(cloned),
|
||||||
|
ApiError::from(StorageError::from(received)),
|
||||||
|
ApiError::from(StorageError::Io(IoError::other(IoError::other(status())))),
|
||||||
|
ApiError::from(IoError::other(status())),
|
||||||
|
ApiError::from(IoError::other(StorageError::from(status()))),
|
||||||
|
] {
|
||||||
|
assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
|
||||||
|
assert_eq!(error.diagnostic().rpc_code, Some(tonic::Code::ResourceExhausted));
|
||||||
|
assert_eq!(S3Error::from(error).status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_admission_marker_requires_its_exact_value_and_resource_code() {
|
||||||
|
for code in [
|
||||||
|
tonic::Code::Unauthenticated,
|
||||||
|
tonic::Code::Internal,
|
||||||
|
tonic::Code::ResourceExhausted,
|
||||||
|
] {
|
||||||
|
for marker in ["replay-cache", "unknown-admission"] {
|
||||||
|
if code == tonic::Code::ResourceExhausted && marker == "replay-cache" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut status = tonic::Status::new(code, "RPC replay admission exhausted");
|
||||||
|
status
|
||||||
|
.metadata_mut()
|
||||||
|
.insert(RPC_ADMISSION_METADATA_KEY, marker.parse().expect("valid test marker"));
|
||||||
|
let error = ApiError::from(StorageError::from(status));
|
||||||
|
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let error = ApiError::from(IoError::other("RPC replay admission exhausted"));
|
||||||
|
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() {
|
fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() {
|
||||||
for code in [
|
for code in [
|
||||||
|
|||||||
@@ -614,22 +614,6 @@ mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[serial]
|
|
||||||
fn writable_node_readiness_keeps_200_for_transient_pool_metadata_timeout() {
|
|
||||||
let mut report = ready_report();
|
|
||||||
report.degraded_reasons = vec![ReadinessDegradedReason::PoolMetadataCheckTimeout];
|
|
||||||
|
|
||||||
with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
|
|
||||||
let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs", None, None);
|
|
||||||
|
|
||||||
assert_eq!(parts.status_code, StatusCode::OK);
|
|
||||||
let payload = parts.payload.expect("GET readiness body");
|
|
||||||
assert_eq!(payload["ready"], true);
|
|
||||||
assert_eq!(payload["degradedReasons"], json!(["pool_metadata_check_timeout"]));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn readiness_collects_object_stalls_and_recovers_on_completion() {
|
async fn readiness_collects_object_stalls_and_recovers_on_completion() {
|
||||||
let object_traffic_health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
|
let object_traffic_health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
|
||||||
|
|||||||
@@ -3405,6 +3405,16 @@ fn hyper_body_write_close_kind(debug: &str) -> Option<&'static str> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rpc_auth_failure_status(error: &std::io::Error) -> Status {
|
||||||
|
if storage::rpc_auth_is_resource_exhausted(error) {
|
||||||
|
crate::error::rpc_replay_admission_exhausted_status()
|
||||||
|
} else if storage::tonic_rpc_auth_failure_reason(error) == "stale_boot_epoch" {
|
||||||
|
Status::unavailable("RPC boot epoch changed")
|
||||||
|
} else {
|
||||||
|
Status::unauthenticated("No valid auth token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::result_large_err)]
|
#[allow(clippy::result_large_err)]
|
||||||
fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
||||||
let local_node_name =
|
let local_node_name =
|
||||||
@@ -3467,6 +3477,12 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
|||||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||||
failure_reason,
|
failure_reason,
|
||||||
);
|
);
|
||||||
|
let rejection = rpc_auth_failure_status(&e);
|
||||||
|
if rejection.code() == tonic::Code::ResourceExhausted {
|
||||||
|
// Existing scoped counters describe overload without producing an
|
||||||
|
// authentication error log for every otherwise valid request.
|
||||||
|
return rejection;
|
||||||
|
}
|
||||||
if failure_reason == "stale_boot_epoch" {
|
if failure_reason == "stale_boot_epoch" {
|
||||||
// The signature is valid, but the peer restarted before this request. Reject it
|
// The signature is valid, but the peer restarted before this request. Reject it
|
||||||
// before execution and let the client retry after its authenticated epoch refresh.
|
// before execution and let the client retry after its authenticated epoch refresh.
|
||||||
@@ -3485,7 +3501,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
|||||||
error = %e,
|
error = %e,
|
||||||
"RPC boot epoch changed; request rejected for retry"
|
"RPC boot epoch changed; request rejected for retry"
|
||||||
);
|
);
|
||||||
Status::unavailable("RPC boot epoch changed")
|
rejection
|
||||||
} else {
|
} else {
|
||||||
error!(
|
error!(
|
||||||
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
|
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
|
||||||
@@ -3502,7 +3518,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
|
|||||||
error = %e,
|
error = %e,
|
||||||
"RPC signature verification failed"
|
"RPC signature verification failed"
|
||||||
);
|
);
|
||||||
Status::unauthenticated("No valid auth token")
|
rejection
|
||||||
}
|
}
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
@@ -7001,6 +7017,35 @@ mod tests {
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rpc_auth_resource_exhaustion_does_not_report_invalid_credentials() {
|
||||||
|
for source in [
|
||||||
|
storage::RpcReplayCacheAdmissionError::Capacity,
|
||||||
|
storage::RpcReplayCacheAdmissionError::MemoryBudget,
|
||||||
|
] {
|
||||||
|
let error = std::io::Error::new(std::io::ErrorKind::WouldBlock, source);
|
||||||
|
let status = rpc_auth_failure_status(&error);
|
||||||
|
assert_eq!(status.code(), tonic::Code::ResourceExhausted);
|
||||||
|
let api = crate::error::ApiError::from(crate::storage_api::error::StorageError::from(status));
|
||||||
|
let response = s3s::S3Error::from(api);
|
||||||
|
assert_eq!(response.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||||
|
}
|
||||||
|
for message in [
|
||||||
|
"RPC request replay detected",
|
||||||
|
"Invalid RPC v2 signature",
|
||||||
|
"Missing signature header",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
rpc_auth_failure_status(&std::io::Error::other(message)).code(),
|
||||||
|
tonic::Code::Unauthenticated
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
rpc_auth_failure_status(&std::io::Error::other("RPC boot epoch is stale")).code(),
|
||||||
|
tonic::Code::Unavailable
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial_test::serial]
|
#[serial_test::serial]
|
||||||
async fn rpc_auth_rejection_records_failure_reason_metric() {
|
async fn rpc_auth_rejection_records_failure_reason_metric() {
|
||||||
|
|||||||
+46
-290
@@ -37,7 +37,7 @@ use std::task::{Context, Poll};
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet},
|
collections::{HashMap, HashSet},
|
||||||
sync::{Mutex as StdMutex, OnceLock},
|
sync::OnceLock,
|
||||||
time::Instant,
|
time::Instant,
|
||||||
};
|
};
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
@@ -80,7 +80,6 @@ const LOG_COMPONENT_SERVER: &str = "server";
|
|||||||
const LOG_SUBSYSTEM_READINESS: &str = "readiness";
|
const LOG_SUBSYSTEM_READINESS: &str = "readiness";
|
||||||
const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
|
const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
|
||||||
const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
|
const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
|
||||||
const METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL: &str = "rustfs_pool_metadata_check_timeouts_total";
|
|
||||||
|
|
||||||
pub use crate::shared_types::{
|
pub use crate::shared_types::{
|
||||||
DependencyReadiness, DependencyReadinessReport, MissingReadinessDrive, ReadinessDegradedReason, StorageReadinessDetails,
|
DependencyReadiness, DependencyReadinessReport, MissingReadinessDrive, ReadinessDegradedReason, StorageReadinessDetails,
|
||||||
@@ -293,66 +292,6 @@ struct StorageWriteReadinessStatus {
|
|||||||
pool_metadata_reason: Option<ReadinessDegradedReason>,
|
pool_metadata_reason: Option<ReadinessDegradedReason>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Default)]
|
|
||||||
struct NodePoolMetadataReadinessCache {
|
|
||||||
entry: Option<NodePoolMetadataReadinessCacheEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
|
||||||
struct NodePoolMetadataReadinessCacheEntry {
|
|
||||||
cached_at: Instant,
|
|
||||||
observed_at: Instant,
|
|
||||||
status: StorageWriteReadinessStatus,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl NodePoolMetadataReadinessCache {
|
|
||||||
fn observe(
|
|
||||||
&mut self,
|
|
||||||
observed: StorageWriteReadinessStatus,
|
|
||||||
observed_at: Instant,
|
|
||||||
now: Instant,
|
|
||||||
ttl: Duration,
|
|
||||||
) -> StorageWriteReadinessStatus {
|
|
||||||
if ttl.is_zero() {
|
|
||||||
self.entry = None;
|
|
||||||
return observed;
|
|
||||||
}
|
|
||||||
|
|
||||||
if observed.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetadataCheckTimeout) {
|
|
||||||
let Some(previous) = self.entry else {
|
|
||||||
return observed;
|
|
||||||
};
|
|
||||||
if now.saturating_duration_since(previous.cached_at) > ttl {
|
|
||||||
return observed;
|
|
||||||
}
|
|
||||||
|
|
||||||
// A confirmed write block must never be hidden by a later timeout.
|
|
||||||
if previous.status.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetaWriteBlocked) {
|
|
||||||
return previous.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
// A transient inspection outage is diagnostic, not proof that the
|
|
||||||
// writer became unsafe. Keep the last confirmed writable state.
|
|
||||||
if previous.status.ready {
|
|
||||||
return StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return observed;
|
|
||||||
}
|
|
||||||
|
|
||||||
if self.entry.is_none_or(|entry| entry.observed_at <= observed_at) {
|
|
||||||
self.entry = Some(NodePoolMetadataReadinessCacheEntry {
|
|
||||||
cached_at: now,
|
|
||||||
observed_at,
|
|
||||||
status: observed,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
observed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn pool_metadata_write_readiness(result: Result<(), StorageError>) -> StorageWriteReadinessStatus {
|
fn pool_metadata_write_readiness(result: Result<(), StorageError>) -> StorageWriteReadinessStatus {
|
||||||
match result {
|
match result {
|
||||||
Ok(()) => StorageWriteReadinessStatus {
|
Ok(()) => StorageWriteReadinessStatus {
|
||||||
@@ -360,13 +299,9 @@ fn pool_metadata_write_readiness(result: Result<(), StorageError>) -> StorageWri
|
|||||||
pool_metadata_reason: None,
|
pool_metadata_reason: None,
|
||||||
},
|
},
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let pool_metadata_reason = if error.pool_metadata_failure().is_some() {
|
let pool_metadata_reason = error
|
||||||
Some(ReadinessDegradedReason::PoolMetaWriteBlocked)
|
.pool_metadata_failure()
|
||||||
} else if matches!(error, StorageError::Timeout) {
|
.map(|_| ReadinessDegradedReason::PoolMetaWriteBlocked);
|
||||||
Some(ReadinessDegradedReason::PoolMetadataCheckTimeout)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
StorageWriteReadinessStatus {
|
StorageWriteReadinessStatus {
|
||||||
ready: false,
|
ready: false,
|
||||||
pool_metadata_reason,
|
pool_metadata_reason,
|
||||||
@@ -434,11 +369,6 @@ fn health_cluster_timeout() -> Duration {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn node_pool_metadata_readiness_cache() -> &'static StdMutex<NodePoolMetadataReadinessCache> {
|
|
||||||
static CACHE: OnceLock<StdMutex<NodePoolMetadataReadinessCache>> = OnceLock::new();
|
|
||||||
CACHE.get_or_init(|| StdMutex::new(NodePoolMetadataReadinessCache::default()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry>> {
|
fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry>> {
|
||||||
static CACHE: OnceLock<Mutex<Option<StorageReadinessCacheEntry>>> = OnceLock::new();
|
static CACHE: OnceLock<Mutex<Option<StorageReadinessCacheEntry>>> = OnceLock::new();
|
||||||
CACHE.get_or_init(|| Mutex::new(None))
|
CACHE.get_or_init(|| Mutex::new(None))
|
||||||
@@ -543,26 +473,6 @@ async fn update_storage_readiness_cache(status: StorageWriteReadinessStatus) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn apply_node_pool_metadata_timeout_policy(
|
|
||||||
observed: StorageWriteReadinessStatus,
|
|
||||||
observed_at: Instant,
|
|
||||||
) -> StorageWriteReadinessStatus {
|
|
||||||
if observed.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetadataCheckTimeout) {
|
|
||||||
counter!(METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL).increment(1);
|
|
||||||
}
|
|
||||||
let mut cache = node_pool_metadata_readiness_cache()
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
||||||
cache.observe(observed, observed_at, Instant::now(), health_readiness_cache_ttl())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
fn reset_node_pool_metadata_readiness_cache() {
|
|
||||||
*node_pool_metadata_readiness_cache()
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = NodePoolMetadataReadinessCache::default();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn load_cached_lock_quorum_status() -> Option<LockQuorumObservation> {
|
async fn load_cached_lock_quorum_status() -> Option<LockQuorumObservation> {
|
||||||
let ttl = health_readiness_cache_ttl();
|
let ttl = health_readiness_cache_ttl();
|
||||||
if ttl.is_zero() {
|
if ttl.is_zero() {
|
||||||
@@ -855,9 +765,6 @@ fn record_readiness_report(report: &DependencyReadinessReport) {
|
|||||||
&& report.readiness.peer_health_ready;
|
&& report.readiness.peer_health_ready;
|
||||||
gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
|
gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
|
||||||
for reason in &report.degraded_reasons {
|
for reason in &report.degraded_reasons {
|
||||||
if ready && *reason == ReadinessDegradedReason::PoolMetadataCheckTimeout {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
|
counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -944,7 +851,7 @@ pub async fn collect_node_readiness_report() -> DependencyReadinessReport {
|
|||||||
let mut storage_check_timed_out = false;
|
let mut storage_check_timed_out = false;
|
||||||
if let Some(store) = runtime_sources::current_object_store_handle() {
|
if let Some(store) = runtime_sources::current_object_store_handle() {
|
||||||
let pool_metadata_status = store.pool_meta_write_status().await;
|
let pool_metadata_status = store.pool_meta_write_status().await;
|
||||||
storage = apply_node_pool_metadata_timeout_policy(pool_metadata_write_readiness(pool_metadata_status), Instant::now());
|
storage = pool_metadata_write_readiness(pool_metadata_status);
|
||||||
match node_storage_snapshot(store.as_ref(), &lock_observation.online_hosts).await {
|
match node_storage_snapshot(store.as_ref(), &lock_observation.online_hosts).await {
|
||||||
Ok((info, unavailable_drives, missing_drives)) => {
|
Ok((info, unavailable_drives, missing_drives)) => {
|
||||||
details.unavailable_drives = unavailable_drives;
|
details.unavailable_drives = unavailable_drives;
|
||||||
@@ -2927,7 +2834,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pool_metadata_observation_distinguishes_block_timeout_and_unavailable() {
|
fn pool_metadata_observation_distinguishes_block_and_unavailable() {
|
||||||
let blocked = blocked_pool_metadata_status();
|
let blocked = blocked_pool_metadata_status();
|
||||||
assert!(!blocked.ready);
|
assert!(!blocked.ready);
|
||||||
assert_eq!(blocked.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetaWriteBlocked));
|
assert_eq!(blocked.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetaWriteBlocked));
|
||||||
@@ -2935,227 +2842,76 @@ mod tests {
|
|||||||
let writable = pool_metadata_write_readiness(Ok(()));
|
let writable = pool_metadata_write_readiness(Ok(()));
|
||||||
assert!(writable.ready);
|
assert!(writable.ready);
|
||||||
assert_eq!(writable.pool_metadata_reason, None);
|
assert_eq!(writable.pool_metadata_reason, None);
|
||||||
let timed_out = pool_metadata_write_readiness(Err(StorageError::Timeout));
|
|
||||||
assert!(!timed_out.ready);
|
|
||||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
|
||||||
let unavailable = pool_metadata_write_readiness(Err(StorageError::other("pool metadata writes remain blocked")));
|
let unavailable = pool_metadata_write_readiness(Err(StorageError::other("pool metadata writes remain blocked")));
|
||||||
assert!(!unavailable.ready);
|
assert!(!unavailable.ready);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
unavailable.pool_metadata_reason, None,
|
unavailable.pool_metadata_reason, None,
|
||||||
"error text alone must not be interpreted as a typed write block"
|
"error text alone must not be interpreted as a typed write block"
|
||||||
);
|
);
|
||||||
|
|
||||||
let readiness = DependencyReadiness {
|
|
||||||
storage_ready: false,
|
|
||||||
iam_ready: true,
|
|
||||||
lock_quorum_ready: true,
|
|
||||||
peer_health_ready: true,
|
|
||||||
};
|
|
||||||
let report = dependency_readiness_report_from_write_status(readiness, timed_out);
|
|
||||||
assert!(!report.readiness.storage_ready, "inspection timeout must remain fail-closed");
|
|
||||||
assert_eq!(report.degraded_reasons, vec![ReadinessDegradedReason::PoolMetadataCheckTimeout]);
|
|
||||||
assert_eq!(report.degraded_reasons[0].as_str(), "pool_metadata_check_timeout");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
fn readiness_report_records_ready_gauge_and_degraded_reasons() {
|
||||||
fn sticky_pool_metadata_timeout_uses_dedicated_metric_without_degraded_count() {
|
let record = |report: DependencyReadinessReport| {
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
let recorder = DebuggingRecorder::new();
|
let recorder = DebuggingRecorder::new();
|
||||||
let snapshotter = recorder.snapshotter();
|
let snapshotter = recorder.snapshotter();
|
||||||
|
metrics::with_local_recorder(&recorder, || record_readiness_report(&report));
|
||||||
metrics::with_local_recorder(&recorder, || {
|
snapshotter.snapshot().into_vec()
|
||||||
with_var(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"), || {
|
|
||||||
let writable = StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
};
|
};
|
||||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
let ready = DependencyReadiness {
|
||||||
let sticky = apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
Instant::now(),
|
|
||||||
);
|
|
||||||
assert!(sticky.ready);
|
|
||||||
record_readiness_report(&DependencyReadinessReport {
|
|
||||||
readiness: DependencyReadiness {
|
|
||||||
storage_ready: true,
|
storage_ready: true,
|
||||||
iam_ready: true,
|
iam_ready: true,
|
||||||
lock_quorum_ready: true,
|
lock_quorum_ready: true,
|
||||||
peer_health_ready: true,
|
peer_health_ready: true,
|
||||||
},
|
};
|
||||||
degraded_reasons: vec![ReadinessDegradedReason::PoolMetadataCheckTimeout],
|
|
||||||
|
let entries = record(DependencyReadinessReport {
|
||||||
|
readiness: ready,
|
||||||
|
degraded_reasons: Vec::new(),
|
||||||
storage_details: None,
|
storage_details: None,
|
||||||
});
|
});
|
||||||
});
|
let gauge = entries.iter().find_map(|(composite, _, _, value)| {
|
||||||
});
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
|
|
||||||
let entries = snapshotter.snapshot().into_vec();
|
|
||||||
let ready = entries.iter().find_map(|(composite, _, _, value)| {
|
|
||||||
(composite.kind() == MetricKind::Gauge && composite.key().name() == METRIC_RUNTIME_READINESS_READY).then_some(value)
|
(composite.kind() == MetricKind::Gauge && composite.key().name() == METRIC_RUNTIME_READINESS_READY).then_some(value)
|
||||||
});
|
});
|
||||||
assert!(matches!(ready, Some(DebugValue::Gauge(value)) if value.into_inner() == 1.0));
|
assert!(
|
||||||
|
matches!(gauge, Some(DebugValue::Gauge(value)) if value.into_inner() == 1.0),
|
||||||
|
"a ready report must set the readiness gauge: {gauge:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
entries
|
||||||
|
.iter()
|
||||||
|
.all(|(composite, _, _, _)| composite.key().name() != METRIC_RUNTIME_READINESS_DEGRADED_TOTAL),
|
||||||
|
"a ready report without reasons must not count degradation"
|
||||||
|
);
|
||||||
|
|
||||||
|
let entries = record(DependencyReadinessReport {
|
||||||
|
readiness: DependencyReadiness {
|
||||||
|
storage_ready: false,
|
||||||
|
..ready
|
||||||
|
},
|
||||||
|
degraded_reasons: vec![ReadinessDegradedReason::PoolMetaWriteBlocked],
|
||||||
|
storage_details: None,
|
||||||
|
});
|
||||||
|
let gauge = entries.iter().find_map(|(composite, _, _, value)| {
|
||||||
|
(composite.kind() == MetricKind::Gauge && composite.key().name() == METRIC_RUNTIME_READINESS_READY).then_some(value)
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
matches!(gauge, Some(DebugValue::Gauge(value)) if value.into_inner() == 0.0),
|
||||||
|
"a degraded report must clear the readiness gauge: {gauge:?}"
|
||||||
|
);
|
||||||
let degraded = entries.iter().find_map(|(composite, _, _, value)| {
|
let degraded = entries.iter().find_map(|(composite, _, _, value)| {
|
||||||
(composite.kind() == MetricKind::Counter
|
(composite.kind() == MetricKind::Counter
|
||||||
&& composite.key().name() == METRIC_RUNTIME_READINESS_DEGRADED_TOTAL
|
&& composite.key().name() == METRIC_RUNTIME_READINESS_DEGRADED_TOTAL
|
||||||
&& composite
|
&& composite
|
||||||
.key()
|
.key()
|
||||||
.labels()
|
.labels()
|
||||||
.any(|label| label.key() == "reason" && label.value() == "pool_metadata_check_timeout"))
|
.any(|label| label.key() == "reason" && label.value() == "pool_meta_write_blocked"))
|
||||||
.then_some(value)
|
.then_some(value)
|
||||||
});
|
});
|
||||||
assert!(degraded.is_none());
|
assert!(
|
||||||
|
matches!(degraded, Some(DebugValue::Counter(1))),
|
||||||
let timeout = entries.iter().find_map(|(composite, _, _, value)| {
|
"each degraded reason must be counted once: {degraded:?}"
|
||||||
(composite.kind() == MetricKind::Counter && composite.key().name() == METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL)
|
|
||||||
.then_some(value)
|
|
||||||
});
|
|
||||||
assert!(matches!(timeout, Some(DebugValue::Counter(1))));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn node_pool_metadata_timeout_preserves_recent_writable_observation() {
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
|
||||||
let writable = StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
};
|
|
||||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
|
||||||
|
|
||||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
Instant::now(),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(timed_out.ready);
|
|
||||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn node_pool_metadata_timeout_fails_closed_with_empty_cache() {
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
|
||||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
Instant::now(),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(!timed_out.ready);
|
|
||||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn node_pool_metadata_timeout_fails_closed_after_cache_ttl() {
|
|
||||||
let mut cache = NodePoolMetadataReadinessCache::default();
|
|
||||||
let observed_at = Instant::now();
|
|
||||||
let ttl = Duration::from_secs(60);
|
|
||||||
let writable = StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
};
|
|
||||||
assert_eq!(cache.observe(writable, observed_at, observed_at, ttl), writable);
|
|
||||||
|
|
||||||
let timed_out = cache.observe(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
observed_at + Duration::from_secs(61),
|
|
||||||
observed_at + Duration::from_secs(61),
|
|
||||||
ttl,
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(!timed_out.ready);
|
|
||||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn node_pool_metadata_timeout_never_hides_confirmed_block() {
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
|
||||||
let blocked = StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetaWriteBlocked),
|
|
||||||
};
|
|
||||||
let newer_observation = Instant::now();
|
|
||||||
assert_eq!(apply_node_pool_metadata_timeout_policy(blocked, newer_observation), blocked);
|
|
||||||
assert_eq!(
|
|
||||||
apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
},
|
|
||||||
newer_observation
|
|
||||||
.checked_sub(Duration::from_secs(1))
|
|
||||||
.expect("test instant has elapsed time"),
|
|
||||||
),
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
Instant::now(),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(timed_out, blocked);
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn node_pool_metadata_timeout_fails_closed_without_fresh_observation() {
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
|
||||||
let writable = StorageWriteReadinessStatus {
|
|
||||||
ready: true,
|
|
||||||
pool_metadata_reason: None,
|
|
||||||
};
|
|
||||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
|
||||||
|
|
||||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("0"))], async {
|
|
||||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
|
||||||
StorageWriteReadinessStatus {
|
|
||||||
ready: false,
|
|
||||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
|
||||||
},
|
|
||||||
Instant::now(),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(!timed_out.ready);
|
|
||||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
reset_node_pool_metadata_readiness_cache();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ pub enum ReadinessDegradedReason {
|
|||||||
ObjectReadStalled,
|
ObjectReadStalled,
|
||||||
ObjectWriteStalled,
|
ObjectWriteStalled,
|
||||||
PoolMetaWriteBlocked,
|
PoolMetaWriteBlocked,
|
||||||
PoolMetadataCheckTimeout,
|
|
||||||
StorageReadinessCheckTimeout,
|
StorageReadinessCheckTimeout,
|
||||||
ClusterHealthTimeout,
|
ClusterHealthTimeout,
|
||||||
PeerHealthUnavailable,
|
PeerHealthUnavailable,
|
||||||
@@ -65,7 +64,6 @@ impl ReadinessDegradedReason {
|
|||||||
ReadinessDegradedReason::ObjectReadStalled => "object_read_stalled",
|
ReadinessDegradedReason::ObjectReadStalled => "object_read_stalled",
|
||||||
ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled",
|
ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled",
|
||||||
ReadinessDegradedReason::PoolMetaWriteBlocked => "pool_meta_write_blocked",
|
ReadinessDegradedReason::PoolMetaWriteBlocked => "pool_meta_write_blocked",
|
||||||
ReadinessDegradedReason::PoolMetadataCheckTimeout => "pool_metadata_check_timeout",
|
|
||||||
ReadinessDegradedReason::StorageReadinessCheckTimeout => "storage_readiness_check_timeout",
|
ReadinessDegradedReason::StorageReadinessCheckTimeout => "storage_readiness_check_timeout",
|
||||||
ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout",
|
ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout",
|
||||||
ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable",
|
ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable",
|
||||||
|
|||||||
@@ -1664,6 +1664,9 @@ async fn handle_put_file(req: Request<Incoming>, require_auth: bool) -> Response
|
|||||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
if crate::storage::storage_api::rpc_auth_is_resource_exhausted(&e) {
|
||||||
|
return response_with_status(StatusCode::SERVICE_UNAVAILABLE, "RPC replay admission exhausted");
|
||||||
|
}
|
||||||
log_internode_rpc_response_failure!(
|
log_internode_rpc_response_failure!(
|
||||||
StatusCode::FORBIDDEN,
|
StatusCode::FORBIDDEN,
|
||||||
&path,
|
&path,
|
||||||
|
|||||||
@@ -621,7 +621,7 @@ pub(crate) mod ecstore_rpc {
|
|||||||
KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION, NetworkPeerProbeClient,
|
KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION, NetworkPeerProbeClient,
|
||||||
NetworkPeerProbeError, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
NetworkPeerProbeError, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
|
||||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, check_and_record_signed_rpc_nonce,
|
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, check_and_record_signed_rpc_nonce,
|
||||||
decode_heal_bucket_rpc_options, normalize_tonic_rpc_audience, rpc_background_admission,
|
decode_heal_bucket_rpc_options, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
sign_ns_scanner_capability_with_checkpoint_coverage, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||||
verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
||||||
@@ -630,8 +630,8 @@ pub(crate) mod ecstore_rpc {
|
|||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||||
ScannerScopedDirtyUsageAckEntry, build_put_file_auth_trailer, gen_signature_headers, gen_tonic_replay_scope_headers,
|
RpcReplayCacheAdmissionError, ScannerScopedDirtyUsageAckEntry, build_put_file_auth_trailer, gen_signature_headers,
|
||||||
gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_put_file_capability,
|
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_put_file_capability,
|
||||||
verify_tonic_boot_epoch_response, verify_tonic_rpc_response_proof,
|
verify_tonic_boot_epoch_response, verify_tonic_rpc_response_proof,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -2433,6 +2433,10 @@ pub(crate) fn verify_tonic_rpc_signature_with_bootstrap(
|
|||||||
ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
|
ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn rpc_auth_is_resource_exhausted(error: &std::io::Error) -> bool {
|
||||||
|
ecstore_rpc::rpc_auth_is_resource_exhausted(error)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
|
pub(crate) fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
|
||||||
ecstore_rpc::tonic_rpc_auth_failure_reason(error)
|
ecstore_rpc::tonic_rpc_auth_failure_reason(error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -186,10 +186,13 @@ pub(crate) mod server {
|
|||||||
pub(crate) use crate::storage::storage_api::ecstore_disk::{
|
pub(crate) use crate::storage::storage_api::ecstore_disk::{
|
||||||
DiskAPI, DiskOption, new_disk, with_background_resource_unit,
|
DiskAPI, DiskOption, new_disk, with_background_resource_unit,
|
||||||
};
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use crate::storage::storage_api::ecstore_rpc::RpcReplayCacheAdmissionError;
|
||||||
pub(crate) use crate::storage::storage_api::{
|
pub(crate) use crate::storage::storage_api::{
|
||||||
RemoteBackgroundIo, ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, rpc_background_admission,
|
RemoteBackgroundIo, ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
||||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
rpc_auth_is_resource_exhausted, rpc_background_admission, tonic_boot_epoch_challenge,
|
||||||
verify_tonic_rpc_signature_with_bootstrap, with_remote_background_io,
|
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap,
|
||||||
|
with_remote_background_io,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||||
|
|||||||
@@ -181,10 +181,6 @@ const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json";
|
|||||||
const TABLE_RENAME_ROOT: &str = "renames";
|
const TABLE_RENAME_ROOT: &str = "renames";
|
||||||
const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64;
|
const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64;
|
||||||
const TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS: usize = 4096;
|
const TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS: usize = 4096;
|
||||||
/// Listing pages an index-miss scan may consume before failing closed. Only
|
|
||||||
/// table entries count toward the object limit, so maintenance reports and
|
|
||||||
/// other catalog objects cannot trip it; this bounds the listing work itself.
|
|
||||||
const TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_PAGES: usize = 64;
|
|
||||||
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
|
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
|
||||||
const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json";
|
const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json";
|
||||||
const MAINTENANCE_ROOT: &str = "maintenance";
|
const MAINTENANCE_ROOT: &str = "maintenance";
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ pub(super) use backup_types::{TableCatalogBackupStatus, TableCatalogRestoreStatu
|
|||||||
use manifest::table_catalog_backing_manifest;
|
use manifest::table_catalog_backing_manifest;
|
||||||
pub(crate) use object::ObjectTableCatalogStore;
|
pub(crate) use object::ObjectTableCatalogStore;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(super) use object::{bounded_table_entry_objects_for_data_plane_scan, maintenance_subtree_skip_cursor};
|
pub(super) use object::bounded_table_entry_objects_for_data_plane_scan;
|
||||||
|
|
||||||
struct PeriodicTableMaintenanceQueueGuard<'a> {
|
struct PeriodicTableMaintenanceQueueGuard<'a> {
|
||||||
table_id: &'a str,
|
table_id: &'a str,
|
||||||
@@ -1055,6 +1055,20 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET-equivalent table ownership entries, selected before metadata reads.
|
||||||
|
/// Report history must not consume pages. No ordinary S3 list fallback is safe.
|
||||||
|
async fn list_table_entry_objects_page(
|
||||||
|
&self,
|
||||||
|
_bucket: &str,
|
||||||
|
_prefix: &str,
|
||||||
|
_start_after: Option<&str>,
|
||||||
|
_limit: NonZeroUsize,
|
||||||
|
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
|
||||||
|
Err(TableCatalogStoreError::Unsupported(
|
||||||
|
"authoritative table-entry pagination is not supported by this backend".to_string(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
/// GET-equivalent, bounded enumeration for publishing authoritative indexes.
|
/// GET-equivalent, bounded enumeration for publishing authoritative indexes.
|
||||||
/// Backends without this contract must not fall back to an ordinary S3 list.
|
/// Backends without this contract must not fall back to an ordinary S3 list.
|
||||||
async fn list_catalog_objects_page(
|
async fn list_catalog_objects_page(
|
||||||
@@ -2615,6 +2629,33 @@ where
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn list_table_entry_objects_page(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: &str,
|
||||||
|
start_after: Option<&str>,
|
||||||
|
limit: NonZeroUsize,
|
||||||
|
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
|
||||||
|
let max_keys = i32::try_from(limit.get())
|
||||||
|
.map_err(|_| TableCatalogStoreError::Invalid("catalog page size exceeds storage API range".to_string()))?;
|
||||||
|
let page = self
|
||||||
|
.store
|
||||||
|
.clone()
|
||||||
|
.list_identifier_tree_leaves_authoritative(
|
||||||
|
bucket,
|
||||||
|
prefix,
|
||||||
|
TABLE_ENTRY_FILE,
|
||||||
|
start_after.map(str::to_string),
|
||||||
|
max_keys,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|err| storage_error_to_catalog("list authoritative table entries", err))?;
|
||||||
|
Ok(TableCatalogObjectListPage {
|
||||||
|
objects: page.objects.into_iter().map(|object| object.name).collect(),
|
||||||
|
is_truncated: page.is_truncated,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn list_catalog_objects_page(
|
async fn list_catalog_objects_page(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -430,20 +430,6 @@ pub(in crate::table_catalog) fn bounded_table_entry_objects_for_data_plane_scan(
|
|||||||
Ok(entries)
|
Ok(entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// When `object` lies in a table's `maintenance/` subtree, the cursor that skips
|
|
||||||
/// the rest of that subtree. Only keys under that directory are skipped, never
|
|
||||||
/// the table entry or any other table beside it.
|
|
||||||
pub(in crate::table_catalog) fn maintenance_subtree_skip_cursor(prefix: &str, object: &str) -> Option<String> {
|
|
||||||
let relative = object.strip_prefix(prefix)?;
|
|
||||||
let mut segments = relative.splitn(5, '/');
|
|
||||||
let namespace = segments.next()?;
|
|
||||||
let root = segments.next()?;
|
|
||||||
let table = segments.next()?;
|
|
||||||
let maintenance = segments.next()?;
|
|
||||||
(root == TABLE_ROOT && maintenance == MAINTENANCE_ROOT && segments.next().is_some())
|
|
||||||
.then(|| format!("{prefix}{namespace}/{root}/{table}/{maintenance}/\u{10ffff}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct ObjectTableCatalogStore<B> {
|
pub(crate) struct ObjectTableCatalogStore<B> {
|
||||||
pub(in crate::table_catalog) backend: B,
|
pub(in crate::table_catalog) backend: B,
|
||||||
@@ -2861,41 +2847,21 @@ where
|
|||||||
max_catalog_objects: usize,
|
max_catalog_objects: usize,
|
||||||
) -> TableCatalogStoreResult<Vec<String>> {
|
) -> TableCatalogStoreResult<Vec<String>> {
|
||||||
let mut objects = Vec::new();
|
let mut objects = Vec::new();
|
||||||
let mut entries = 0_usize;
|
|
||||||
let mut pages = 0_usize;
|
|
||||||
let mut cursor: Option<String> = None;
|
let mut cursor: Option<String> = None;
|
||||||
let prefix = self.paths.namespace_entries_prefix(table_bucket);
|
let prefix = self.paths.namespace_entries_prefix(table_bucket);
|
||||||
let limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS)
|
let limit = NonZeroUsize::new(TABLE_CATALOG_LIST_MAX_KEYS)
|
||||||
.ok_or_else(|| TableCatalogStoreError::Internal("table catalog listing page size must be positive".to_string()))?;
|
.ok_or_else(|| TableCatalogStoreError::Internal("table catalog listing page size must be positive".to_string()))?;
|
||||||
loop {
|
loop {
|
||||||
// A page that ends inside a table's maintenance subtree skips the
|
|
||||||
// rest of it, so each entry can waste at most one page: honest
|
|
||||||
// catalogs scale with their entries while a report flood under one
|
|
||||||
// table stays within the fixed budget.
|
|
||||||
if pages >= TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_PAGES.saturating_add(entries) {
|
|
||||||
return Err(TableCatalogStoreError::Unavailable(format!(
|
|
||||||
"table data-plane warehouse-index miss scan exceeds the {TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_PAGES}-page listing budget"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let page = self
|
let page = self
|
||||||
.backend
|
.backend
|
||||||
.list_objects_page(self.catalog_bucket(), &prefix, cursor.as_deref(), limit)
|
.list_table_entry_objects_page(self.catalog_bucket(), &prefix, cursor.as_deref(), limit)
|
||||||
.await?;
|
.await?;
|
||||||
pages += 1;
|
let next_cursor = page.objects.last().cloned();
|
||||||
let next_cursor = page
|
|
||||||
.objects
|
|
||||||
.last()
|
|
||||||
.map(|last| maintenance_subtree_skip_cursor(&prefix, last).unwrap_or_else(|| last.clone()));
|
|
||||||
entries += page
|
|
||||||
.objects
|
|
||||||
.iter()
|
|
||||||
.filter(|object| object.ends_with(TABLE_ENTRY_FILE))
|
|
||||||
.count();
|
|
||||||
objects.extend(page.objects);
|
objects.extend(page.objects);
|
||||||
if !page.is_truncated {
|
if !page.is_truncated {
|
||||||
return bounded_table_entry_objects_for_data_plane_scan(objects, false, max_catalog_objects);
|
return bounded_table_entry_objects_for_data_plane_scan(objects, false, max_catalog_objects);
|
||||||
}
|
}
|
||||||
if entries > max_catalog_objects {
|
if objects.len() > max_catalog_objects {
|
||||||
return bounded_table_entry_objects_for_data_plane_scan(objects, true, max_catalog_objects);
|
return bounded_table_entry_objects_for_data_plane_scan(objects, true, max_catalog_objects);
|
||||||
}
|
}
|
||||||
let Some(next_cursor) = next_cursor else {
|
let Some(next_cursor) = next_cursor else {
|
||||||
|
|||||||
@@ -594,6 +594,29 @@ impl TestCatalogObjectBackend {
|
|||||||
state.list_calls = 0;
|
state.list_calls = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn overridden_list_page(&self, bucket: &str, prefix: &str) -> Option<TableCatalogObjectListPage> {
|
||||||
|
let pause = self
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.list_page_pauses
|
||||||
|
.get_mut(&(bucket.to_string(), prefix.to_string()))
|
||||||
|
.and_then(std::collections::VecDeque::pop_front);
|
||||||
|
if let Some(pause) = pause {
|
||||||
|
pause.started.notify_one();
|
||||||
|
pause.release.notified().await;
|
||||||
|
}
|
||||||
|
let mut state = self.state.lock().await;
|
||||||
|
let page = state
|
||||||
|
.list_page_overrides
|
||||||
|
.get_mut(&(bucket.to_string(), prefix.to_string()))
|
||||||
|
.and_then(std::collections::VecDeque::pop_front);
|
||||||
|
if page.is_some() {
|
||||||
|
state.list_calls += 1;
|
||||||
|
}
|
||||||
|
page
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
pub(crate) async fn write_lock_acquisition_count(&self, bucket: &str, object: &str) -> usize {
|
||||||
self.state
|
self.state
|
||||||
.lock()
|
.lock()
|
||||||
@@ -1231,29 +1254,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
|||||||
start_after: Option<&str>,
|
start_after: Option<&str>,
|
||||||
limit: NonZeroUsize,
|
limit: NonZeroUsize,
|
||||||
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
|
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
|
||||||
let pause = self
|
if let Some(page) = self.overridden_list_page(bucket, prefix).await {
|
||||||
.state
|
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.list_page_pauses
|
|
||||||
.get_mut(&(bucket.to_string(), prefix.to_string()))
|
|
||||||
.and_then(std::collections::VecDeque::pop_front);
|
|
||||||
if let Some(pause) = pause {
|
|
||||||
pause.started.notify_one();
|
|
||||||
pause.release.notified().await;
|
|
||||||
}
|
|
||||||
let injected = {
|
|
||||||
let mut state = self.state.lock().await;
|
|
||||||
let page = state
|
|
||||||
.list_page_overrides
|
|
||||||
.get_mut(&(bucket.to_string(), prefix.to_string()))
|
|
||||||
.and_then(std::collections::VecDeque::pop_front);
|
|
||||||
if page.is_some() {
|
|
||||||
state.list_calls += 1;
|
|
||||||
}
|
|
||||||
page
|
|
||||||
};
|
|
||||||
if let Some(page) = injected {
|
|
||||||
return Ok(page);
|
return Ok(page);
|
||||||
}
|
}
|
||||||
let mut objects = self.list_objects(bucket, prefix).await?;
|
let mut objects = self.list_objects(bucket, prefix).await?;
|
||||||
@@ -1266,6 +1267,34 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
|||||||
Ok(TableCatalogObjectListPage { objects, is_truncated })
|
Ok(TableCatalogObjectListPage { objects, is_truncated })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn list_table_entry_objects_page(
|
||||||
|
&self,
|
||||||
|
bucket: &str,
|
||||||
|
prefix: &str,
|
||||||
|
start_after: Option<&str>,
|
||||||
|
limit: NonZeroUsize,
|
||||||
|
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
|
||||||
|
if let Some(page) = self.overridden_list_page(bucket, prefix).await {
|
||||||
|
return Ok(page);
|
||||||
|
}
|
||||||
|
// This in-memory backend has no erasure replicas. Production selects
|
||||||
|
// candidates before metadata I/O and then proves each key at read quorum.
|
||||||
|
let mut inventory = self.list_objects_for_cleanup(bucket, prefix).await?;
|
||||||
|
inventory.retain(|object| object.key.rsplit('/').next() == Some(TABLE_ENTRY_FILE));
|
||||||
|
if inventory
|
||||||
|
.iter()
|
||||||
|
.any(|object| object.version_id.is_some() || object.delete_marker)
|
||||||
|
{
|
||||||
|
return Err(TableCatalogStoreError::Unavailable("versioned table ownership entry".to_string()));
|
||||||
|
}
|
||||||
|
let mut objects = inventory.into_iter().map(|object| object.key).collect::<Vec<_>>();
|
||||||
|
objects.sort();
|
||||||
|
objects.retain(|object| start_after.is_none_or(|after| object.as_str() > after));
|
||||||
|
let is_truncated = objects.len() >= limit.get();
|
||||||
|
objects.truncate(limit.get());
|
||||||
|
Ok(TableCatalogObjectListPage { objects, is_truncated })
|
||||||
|
}
|
||||||
|
|
||||||
async fn list_objects_for_cleanup(
|
async fn list_objects_for_cleanup(
|
||||||
&self,
|
&self,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
|
|||||||
@@ -1814,43 +1814,6 @@ fn table_data_plane_index_miss_scan_rejects_catalogs_above_its_object_limit() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn table_data_plane_index_miss_scan_skips_only_a_table_maintenance_subtree() {
|
|
||||||
let prefix = "catalog/namespaces/";
|
|
||||||
let skip = |object: &str| maintenance_subtree_skip_cursor(prefix, &format!("{prefix}{object}"));
|
|
||||||
assert_eq!(
|
|
||||||
skip("sales/tables/orders/maintenance/hash/jobs/job.json").as_deref(),
|
|
||||||
Some("catalog/namespaces/sales/tables/orders/maintenance/\u{10ffff}")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
skip("sales/tables/orders/maintenance/hash/config.json").as_deref(),
|
|
||||||
Some("catalog/namespaces/sales/tables/orders/maintenance/\u{10ffff}")
|
|
||||||
);
|
|
||||||
// A table or namespace literally named "maintenance" is not a subtree.
|
|
||||||
assert_eq!(skip("sales/tables/maintenance/table-entry.json"), None);
|
|
||||||
assert_eq!(skip("maintenance/tables/orders/table-entry.json"), None);
|
|
||||||
assert_eq!(
|
|
||||||
skip("maintenance/tables/maintenance/maintenance/hash/jobs/job.json").as_deref(),
|
|
||||||
Some("catalog/namespaces/maintenance/tables/maintenance/maintenance/\u{10ffff}")
|
|
||||||
);
|
|
||||||
// Entries, bridges, views and a bare file named maintenance never jump.
|
|
||||||
assert_eq!(skip("sales/tables/orders/table-entry.json"), None);
|
|
||||||
assert_eq!(skip("sales/tables/orders/external-catalog/bridge.json"), None);
|
|
||||||
assert_eq!(skip("sales/views/report/maintenance/hash/jobs/job.json"), None);
|
|
||||||
assert_eq!(skip("sales/tables/orders/maintenance"), None);
|
|
||||||
assert_eq!(skip("sales/namespace.json"), None);
|
|
||||||
assert_eq!(maintenance_subtree_skip_cursor(prefix, "other/sales/tables/orders/maintenance/x/y"), None);
|
|
||||||
// The jump lands before the entry that follows the subtree.
|
|
||||||
assert!(
|
|
||||||
"catalog/namespaces/sales/tables/orders/maintenance/\u{10ffff}"
|
|
||||||
< "catalog/namespaces/sales/tables/orders/table-entry.json"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
"catalog/namespaces/sales/tables/orders/maintenance/\u{10ffff}"
|
|
||||||
< "catalog/namespaces/sales/tables/orders0/table-entry.json"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn catalog_lock_authority_failures_are_typed_as_unavailable() {
|
fn catalog_lock_authority_failures_are_typed_as_unavailable() {
|
||||||
for error in [
|
for error in [
|
||||||
@@ -6880,6 +6843,8 @@ async fn seed_noise_table_with_job_reports(
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn table_data_plane_index_miss_scan_pages_scale_with_entries_not_job_reports() {
|
async fn table_data_plane_index_miss_scan_pages_scale_with_entries_not_job_reports() {
|
||||||
|
let mut baseline_pages = None;
|
||||||
|
for reports in [0, 80] {
|
||||||
let backend = TestCatalogObjectBackend::default();
|
let backend = TestCatalogObjectBackend::default();
|
||||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||||
let bucket = "analytics";
|
let bucket = "analytics";
|
||||||
@@ -6893,13 +6858,12 @@ async fn table_data_plane_index_miss_scan_pages_scale_with_entries_not_job_repor
|
|||||||
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"))
|
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"))
|
||||||
.await
|
.await
|
||||||
.expect("warehouse index entry should be removed");
|
.expect("warehouse index entry should be removed");
|
||||||
// 1000 honest tables with 80 historical job reports each: ~80 pages of
|
// Keep the same table inventory with and without historical job reports.
|
||||||
// objects, beyond the fixed page budget on its own, yet well within one
|
|
||||||
// extra page per table entry.
|
|
||||||
let noise_namespace = Namespace::parse("noise").expect("namespace should parse");
|
let noise_namespace = Namespace::parse("noise").expect("namespace should parse");
|
||||||
for index in 0..1000 {
|
for index in 0..1000 {
|
||||||
let noise_table = IdentifierSegment::parse(format!("noise-{index:04}")).expect("table should parse");
|
let noise_table = IdentifierSegment::parse(format!("noise-{index:04}")).expect("table should parse");
|
||||||
seed_noise_table_with_job_reports(&store, bucket, &noise_namespace, &noise_table, &format!("noise-{index}"), 80).await;
|
seed_noise_table_with_job_reports(&store, bucket, &noise_namespace, &noise_table, &format!("noise-{index}"), reports)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
backend.reset_call_counts().await;
|
backend.reset_call_counts().await;
|
||||||
|
|
||||||
@@ -6909,10 +6873,93 @@ async fn table_data_plane_index_miss_scan_pages_scale_with_entries_not_job_repor
|
|||||||
.expect("the scan must retain table-aware protection");
|
.expect("the scan must retain table-aware protection");
|
||||||
assert_eq!(resource.table, "orders");
|
assert_eq!(resource.table, "orders");
|
||||||
let pages = backend.list_call_count().await;
|
let pages = backend.list_call_count().await;
|
||||||
|
assert!(pages > 0, "the missing index must exercise enumeration");
|
||||||
|
if let Some(expected) = baseline_pages {
|
||||||
|
assert_eq!(pages, expected, "job history must not consume ownership listing pages");
|
||||||
|
} else {
|
||||||
|
baseline_pages = Some(pages);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn table_entry_scan_preserves_nested_namespaces_named_like_maintenance() {
|
||||||
|
for name in ["sales", "sales.tables.orders.maintenance"] {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse(name).expect("namespace should parse");
|
||||||
|
let owner = IdentifierSegment::parse("zz-owner").expect("owner should parse");
|
||||||
|
let current = default_table_metadata_file_path(&namespace, &owner, "00001.metadata.json");
|
||||||
|
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &owner, current).await;
|
||||||
|
backend
|
||||||
|
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"))
|
||||||
|
.await
|
||||||
|
.expect("remove the owner index to exercise catalog fallback");
|
||||||
|
for index in 0..1000 {
|
||||||
|
let table = IdentifierSegment::parse(format!("table-{index:04}")).expect("table should parse");
|
||||||
|
seed_noise_table_with_job_reports(&store, bucket, &namespace, &table, &format!("id-{index}"), 0).await;
|
||||||
|
}
|
||||||
|
let resource = store
|
||||||
|
.resolve_table_data_plane_resource_for_authorization(bucket, "tables/table-id/data/part.parquet")
|
||||||
|
.await
|
||||||
|
.expect("registered table entries must remain enumerable")
|
||||||
|
.unwrap_or_else(|| panic!("namespace {name} lost its table owner across a page boundary"));
|
||||||
|
assert_eq!(resource.namespace, name);
|
||||||
|
assert_eq!(resource.table, "zz-owner");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn object_maintenance_preserves_an_owner_after_nested_namespace_page_boundary() {
|
||||||
|
let backend = TestCatalogObjectBackend::default();
|
||||||
|
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||||
|
let bucket = "analytics";
|
||||||
|
let namespace = Namespace::parse("sales").expect("namespace should parse");
|
||||||
|
let table = IdentifierSegment::parse("orders").expect("table should parse");
|
||||||
|
let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
|
||||||
|
let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
|
||||||
|
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, current.clone()).await;
|
||||||
|
backend
|
||||||
|
.seed_object(bucket, ¤t, br#"{"metadata-log":[]}"#.to_vec())
|
||||||
|
.await;
|
||||||
|
backend.seed_object(bucket, &orphan, br#"{"metadata-log":[]}"#.to_vec()).await;
|
||||||
|
let report = store
|
||||||
|
.plan_table_metadata_maintenance(bucket, "sales", "orders", 0)
|
||||||
|
.await
|
||||||
|
.expect("the unowned old metadata should initially be a cleanup candidate");
|
||||||
|
assert_eq!(report.deletable_metadata_locations, vec![orphan.clone()]);
|
||||||
|
|
||||||
|
let nested = Namespace::parse("sales.tables.orders.maintenance").expect("nested namespace should parse");
|
||||||
|
store
|
||||||
|
.create_namespace(test_namespace_entry(bucket, &nested))
|
||||||
|
.await
|
||||||
|
.expect("nested namespace should be created");
|
||||||
|
for index in 0..1100 {
|
||||||
|
let noise = IdentifierSegment::parse(format!("table-{index:04}")).expect("table should parse");
|
||||||
|
seed_noise_table_with_job_reports(&store, bucket, &nested, &noise, &format!("noise-{index}"), 0).await;
|
||||||
|
}
|
||||||
|
let owner = IdentifierSegment::parse("zz-owner").expect("owner should parse");
|
||||||
|
let mut other = test_table_entry(bucket, &nested, &owner, orphan.clone());
|
||||||
|
other.table_id = "other-id".to_string();
|
||||||
|
other.table_uuid = "other-uuid".to_string();
|
||||||
|
other.warehouse_location = format!("s3://{bucket}/tables/other-id");
|
||||||
|
store
|
||||||
|
.create_table(other)
|
||||||
|
.await
|
||||||
|
.expect("another table may adopt the metadata before cleanup");
|
||||||
|
let deletes_before = backend.state.lock().await.delete_attempts.clone();
|
||||||
|
let result = store
|
||||||
|
.delete_table_metadata_maintenance_report(bucket, "sales", "orders", report)
|
||||||
|
.await;
|
||||||
assert!(
|
assert!(
|
||||||
pages > TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_PAGES && pages <= TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_PAGES + 1001,
|
backend
|
||||||
"the honest catalog needs {pages} pages: more than the fixed budget, at most one extra per entry"
|
.object_exists(bucket, &orphan)
|
||||||
|
.await
|
||||||
|
.expect("read back the current owner's metadata")
|
||||||
);
|
);
|
||||||
|
assert_eq!(backend.state.lock().await.delete_attempts, deletes_before);
|
||||||
|
assert_matches!(result, Err(TableCatalogStoreError::Conflict(message)) if message.contains("exclusive metadata"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -6930,8 +6977,7 @@ async fn table_data_plane_index_miss_scan_bounds_a_job_report_flood_under_one_ta
|
|||||||
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"))
|
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path(bucket, "tables/table-id/"))
|
||||||
.await
|
.await
|
||||||
.expect("warehouse index entry should be removed");
|
.expect("warehouse index entry should be removed");
|
||||||
// A table-scoped principal floods one table's job history: 70 reports per
|
// Historical report volume must not consume ownership pagination.
|
||||||
// page budget would have exhausted the old fixed budget bucket-wide.
|
|
||||||
for job in 0..70_000 {
|
for job in 0..70_000 {
|
||||||
backend
|
backend
|
||||||
.seed_object(
|
.seed_object(
|
||||||
@@ -6951,7 +6997,7 @@ async fn table_data_plane_index_miss_scan_bounds_a_job_report_flood_under_one_ta
|
|||||||
.expect("the scan must retain table-aware protection");
|
.expect("the scan must retain table-aware protection");
|
||||||
assert_eq!(resource.table, "orders");
|
assert_eq!(resource.table, "orders");
|
||||||
let pages = backend.list_call_count().await;
|
let pages = backend.list_call_count().await;
|
||||||
assert!(pages <= 2, "the page that ends inside the flood skips the rest of it, not {pages} pages");
|
assert!(pages <= 2, "report history must not consume {pages} ownership pages");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Exercise the release download entry point without fetching external tools."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT = Path(__file__).resolve().parents[1] / ".github/actions/setup/download_release_asset.py"
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseAssetTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temp.cleanup)
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
self.body = b"verified tool fixture"
|
||||||
|
self.sha256 = hashlib.sha256(self.body).hexdigest()
|
||||||
|
self.asset = {"name": "tool.zip", "id": 17, "size": len(self.body), "digest": "sha256:" + self.sha256}
|
||||||
|
self.metadata = {"tag_name": "v1.0", "assets": [self.asset]}
|
||||||
|
self.output = self.root / "tool.zip"
|
||||||
|
self.output.write_bytes(b"previous file")
|
||||||
|
self.calls = self.root / "calls.jsonl"
|
||||||
|
gh = self.root / "gh"
|
||||||
|
gh.write_text(f"#!{sys.executable}\n" + '''import json,os,pathlib,sys
|
||||||
|
args=sys.argv[1:]
|
||||||
|
with pathlib.Path(os.environ['CALLS']).open('a') as out:out.write(json.dumps(args)+'\\n')
|
||||||
|
if '/releases/tags/' in args[-1]:
|
||||||
|
if os.environ.get('FAIL_METADATA'):raise SystemExit(19)
|
||||||
|
print(os.environ['METADATA'])
|
||||||
|
else:
|
||||||
|
if os.environ.get('FAIL_DOWNLOAD'):raise SystemExit(23)
|
||||||
|
sys.stdout.buffer.write(bytes.fromhex(os.environ['BODY']))
|
||||||
|
''')
|
||||||
|
gh.chmod(0o755)
|
||||||
|
self.env = dict(os.environ, PATH=str(self.root) + os.pathsep + os.environ["PATH"], CALLS=str(self.calls), GH_TOKEN="synthetic-do-not-log-token")
|
||||||
|
|
||||||
|
def invoke(self, extra=(), **env):
|
||||||
|
environment = dict(self.env, METADATA=json.dumps(self.metadata), BODY=self.body.hex(), **env)
|
||||||
|
return subprocess.run([sys.executable, str(SCRIPT), "--repository", "owner/repo", "--tag", "v1.0", "--asset", "tool.zip", "--output", str(self.output), *extra], env=environment, capture_output=True, text=True)
|
||||||
|
|
||||||
|
def assert_failed_atomically(self, result):
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertEqual(self.output.read_bytes(), b"previous file")
|
||||||
|
self.assertEqual(list(self.root.glob(".release-asset-*")), [])
|
||||||
|
self.assertNotIn(self.env["GH_TOKEN"], result.stdout + result.stderr)
|
||||||
|
|
||||||
|
def test_valid_asset_uses_api_and_installs_only_verified_bytes(self):
|
||||||
|
result = self.invoke(("--sha256", self.sha256))
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(self.output.read_bytes(), self.body)
|
||||||
|
calls = [json.loads(line) for line in self.calls.read_text().splitlines()]
|
||||||
|
self.assertEqual(calls[0], ["api", "--hostname", "github.com", "repos/owner/repo/releases/tags/v1.0"])
|
||||||
|
self.assertEqual(calls[1], ["api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", "repos/owner/repo/releases/assets/17"])
|
||||||
|
self.assertNotIn(self.env["GH_TOKEN"], json.dumps(calls) + result.stdout + result.stderr)
|
||||||
|
|
||||||
|
def test_download_failure_preserves_destination(self):
|
||||||
|
self.assert_failed_atomically(self.invoke(FAIL_DOWNLOAD="1"))
|
||||||
|
|
||||||
|
def test_missing_gh_fails_without_replacing_destination(self):
|
||||||
|
(self.root / "gh").unlink()
|
||||||
|
self.env["PATH"] = str(self.root)
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_metadata_failure_preserves_destination(self):
|
||||||
|
self.assert_failed_atomically(self.invoke(FAIL_METADATA="1"))
|
||||||
|
|
||||||
|
def test_corrupt_download_fails_digest_validation(self):
|
||||||
|
self.body = b"modified tool fixture"
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_truncated_download_fails_size_validation(self):
|
||||||
|
self.asset["size"] += 1
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_pinned_digest_cannot_be_replaced_by_release_metadata(self):
|
||||||
|
self.assert_failed_atomically(self.invoke(("--sha256", "0" * 64)))
|
||||||
|
self.assertEqual(len(self.calls.read_text().splitlines()), 1)
|
||||||
|
|
||||||
|
def test_ambiguous_or_missing_assets_fail_before_download(self):
|
||||||
|
for assets in ([], [self.asset, self.asset]):
|
||||||
|
with self.subTest(assets=len(assets)):
|
||||||
|
self.metadata["assets"] = assets
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_metadata_tag_must_match_requested_version(self):
|
||||||
|
self.metadata["tag_name"] = "v2.0"
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_malformed_release_inventory_fails_before_download(self):
|
||||||
|
for metadata in ([], {"tag_name": "v1.0", "assets": None}, {"tag_name": "v1.0", "assets": [None]}):
|
||||||
|
with self.subTest(metadata=metadata):
|
||||||
|
self.metadata = metadata
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
|
||||||
|
def test_missing_digest_or_invalid_identity_fails_before_download(self):
|
||||||
|
for field, value in [("digest", None), ("digest", "sha256:invalid"), ("id", True), ("id", 0), ("size", 0), ("size", 513 * 1024 * 1024)]:
|
||||||
|
with self.subTest(field=field, value=value):
|
||||||
|
self.asset[field] = value
|
||||||
|
self.assert_failed_atomically(self.invoke())
|
||||||
|
self.asset = {"name": "tool.zip", "id": 17, "size": len(self.body), "digest": "sha256:" + self.sha256}
|
||||||
|
self.metadata["assets"] = [self.asset]
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -206,13 +206,6 @@ grep -q 'RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS: "100"' <<<"$scanner_config_output
|
|||||||
grep -q 'RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE: "1099511627776"' <<<"$scanner_config_output"
|
grep -q 'RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE: "1099511627776"' <<<"$scanner_config_output"
|
||||||
grep -q 'RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS: "65538"' <<<"$scanner_config_output"
|
grep -q 'RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS: "65538"' <<<"$scanner_config_output"
|
||||||
|
|
||||||
target_store_output=$(render_chart --set config.rustfs.target_store_dir=/data/rustfs0/.rustfs)
|
|
||||||
grep -q 'RUSTFS_TARGET_STORE_DIR: "/data/rustfs0/.rustfs"' <<<"$target_store_output"
|
|
||||||
if grep -q 'RUSTFS_TARGET_STORE_DIR' <<<"$(render_chart)"; then
|
|
||||||
echo "RUSTFS_TARGET_STORE_DIR must not be rendered by default" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Fail-closed credential checks. Rendering must fail when no credentials,
|
# Fail-closed credential checks. Rendering must fail when no credentials,
|
||||||
# existingSecret, or allowInsecureDefaults override is supplied.
|
# existingSecret, or allowInsecureDefaults override is supplied.
|
||||||
default_render_status=0
|
default_render_status=0
|
||||||
|
|||||||
Reference in New Issue
Block a user