mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-31 00:06:13 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 46337942d5 | |||
| 46e56019eb | |||
| e02c27feaf | |||
| 440f831d12 | |||
| 0ad3afcf0a | |||
| 331fb89628 | |||
| 2b243d8659 | |||
| c52570d385 | |||
| e0b1635abf | |||
| f0f2b0cf48 | |||
| 0882e99c32 | |||
| 802ac8a1bd | |||
| 9408b90f13 | |||
| 08ac1ec218 | |||
| 388d1f17e6 | |||
| 22c093e66b | |||
| 24ab34a5ee | |||
| 9068d83400 | |||
| 20fc897036 | |||
| a86117590c |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a58501e7f6f1407201170c300f5fe91032a43addd4cb215306cf321ab59a0805
|
||||
sha256-linux=57791d182063049c5f0ca91ae0a6bf64006af6f991a5094e0cb7c8d8491187ed
|
||||
sha256-darwin=1520cd3027c3a6924afc6a204eddd5475bc30f0109d4c2847456a66177263244
|
||||
sha256-linux=26155b7f1c5bf6bf596ff24a4136e6767ea4ee6461c32db7fbe1400791da2c28
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=8ef80b7353d1e0c4f863a2aca7e996d23858b8c32b9edf8fd272076fa63a83d8
|
||||
sha256-linux=99fe1cd128ade2df69b9e3ecf94b91021185a9769293420b1ba5d24760f2ea99
|
||||
sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
|
||||
sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
|
||||
|
||||
@@ -27,6 +27,7 @@ script-tests: ## Run shell script tests
|
||||
trap 'status=$$?; printf "ERROR: script-tests failed (exit %s): %s\n" "$$status" "$$BASH_COMMAND" >&2; exit "$$status"' ERR
|
||||
./scripts/test_build_rustfs_options.sh
|
||||
./scripts/test_docker_runtime_timezone.sh
|
||||
sh ./scripts/test_ca_bundle.sh
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Set up Docker Compose
|
||||
description: Install the verified Compose CLI used by native Linux x86_64 Connect acceptance
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install verified Docker Compose CLI plugin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ $(uname -s) == Linux && $(uname -m) == x86_64 ]]
|
||||
plugin_dir="${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins"
|
||||
mkdir -p "$plugin_dir"
|
||||
compose_download=$(mktemp "${RUNNER_TEMP:?}/connect-compose.XXXXXX")
|
||||
compose_staged=$(mktemp "$plugin_dir/.docker-compose.XXXXXX")
|
||||
trap 'rm -f "$compose_download" "$compose_staged"' EXIT
|
||||
curl --fail --location --retry 3 \
|
||||
https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \
|
||||
--output "$compose_download"
|
||||
printf '%s %s\n' \
|
||||
dba9d98e1ba5bfe11d88c99b9bd32fc4a0624a30fafe68eea34d61a3e42fd372 \
|
||||
"$compose_download" | sha256sum --check --strict
|
||||
install -m 0755 "$compose_download" "$compose_staged"
|
||||
mv -f "$compose_staged" "$plugin_dir/docker-compose"
|
||||
[[ $(docker compose version --short) == 2.40.3 ]]
|
||||
@@ -20,7 +20,6 @@ on:
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
- 'third-party/pyroscope/**'
|
||||
- 'deny.toml'
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
@@ -40,7 +39,6 @@ on:
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
- 'third-party/pyroscope/**'
|
||||
- 'deny.toml'
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
@@ -131,13 +129,6 @@ jobs:
|
||||
- name: Run cargo-deny
|
||||
run: cargo deny check --hide-inclusion-graph advisories sources bans licenses
|
||||
|
||||
- name: Test patched profiling dependency
|
||||
env:
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/target/pyroscope-dependency
|
||||
run: >-
|
||||
cargo test --manifest-path third-party/pyroscope/Cargo.toml
|
||||
--features backend-pprof-rs --lib --test rustfs_zstd
|
||||
|
||||
workflow-pin-report:
|
||||
name: Workflow Pin Report
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
|
||||
@@ -322,7 +322,8 @@ jobs:
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
# Preserve the reviewed 1.0.3 Console in this security-only release branch.
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/tags/v0.1.37"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
@@ -359,12 +360,15 @@ jobs:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
if tag != "v0.1.37":
|
||||
raise SystemExit("Console release does not match the security-release pin")
|
||||
expected_digest = "sha256:e6ff2c7e7695de824ec4cda91d8611d1aba5d8462e6809592c92a50b23f5b9ab"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
if name == "rustfs-console-v0.1.37.zip" and digest == expected_digest and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
|
||||
@@ -57,6 +57,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -167,7 +170,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/diagnostic-schedule-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_DIAGNOSTIC_SCHEDULE_EVIDENCE="$GITHUB_WORKSPACE/diagnostic-schedule-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=diagnostic-schedule
|
||||
|
||||
@@ -57,6 +57,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -204,7 +207,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/health-service-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_HEALTH_SERVICE_EVIDENCE="$GITHUB_WORKSPACE/health-service-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=health-service
|
||||
|
||||
@@ -73,6 +73,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -159,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PERFORMANCE_DRIVE_EVIDENCE="$GITHUB_WORKSPACE/performance-drive-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=performance-drive
|
||||
|
||||
@@ -73,6 +73,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -159,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PERFORMANCE_NETWORK_EVIDENCE="$GITHUB_WORKSPACE/performance-network-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=performance-network
|
||||
|
||||
@@ -77,6 +77,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -178,7 +181,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_CPU_CATALOG="$GITHUB_WORKSPACE/binary/rustfs.cpu-symbol-catalog.json" \
|
||||
CONNECT_E2E_CPU_CATALOG_SHA256="$CATALOG_SHA256" \
|
||||
|
||||
@@ -57,6 +57,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -203,7 +206,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/profile-memory-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PROFILE_MEMORY_EVIDENCE="$GITHUB_WORKSPACE/profile-memory-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=profile-memory
|
||||
|
||||
@@ -81,6 +81,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -206,7 +209,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PROFILE_THREADS_EVIDENCE="$GITHUB_WORKSPACE/profile-threads-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=profile-threads
|
||||
|
||||
@@ -73,6 +73,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -159,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_API_EVIDENCE="$GITHUB_WORKSPACE/top-api-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-api
|
||||
|
||||
@@ -68,6 +68,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -218,7 +221,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_DISK_EVIDENCE="$GITHUB_WORKSPACE/top-disk-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-disk
|
||||
|
||||
@@ -73,6 +73,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -159,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_LOCKS_EVIDENCE="$GITHUB_WORKSPACE/top-locks-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-locks
|
||||
|
||||
@@ -73,6 +73,9 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
@@ -159,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_RPC_EVIDENCE="$GITHUB_WORKSPACE/top-rpc-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-rpc
|
||||
|
||||
Generated
+353
-391
File diff suppressed because it is too large
Load Diff
+81
-87
@@ -67,7 +67,6 @@ members = [
|
||||
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
]
|
||||
exclude = ["third-party/pyroscope"]
|
||||
resolver = "3"
|
||||
|
||||
[workspace.package]
|
||||
@@ -75,7 +74,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.1"
|
||||
version = "1.0.4"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -92,62 +91,62 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.1" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.4" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.4" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.4" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.4" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.4" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.4" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.4" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.4" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.4" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.4" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.4" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.4" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.4" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.4" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.4" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.4" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.4" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.4" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.4" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.4" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.4" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.4" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.4" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.4" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.4" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.4", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.4" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.4" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.4" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.4" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.4" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.4" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.4" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.4" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.4" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.4" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.4" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.4" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.4" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.4" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.4" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.4" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.4" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.4" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.4" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.4" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.4" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.4" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.4" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.4" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.50" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -156,8 +155,8 @@ futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.1" }
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
@@ -170,7 +169,7 @@ minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.2" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
@@ -221,7 +220,7 @@ openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.45" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
x509-parser = "0.18.1"
|
||||
@@ -244,22 +243,22 @@ tokio-postgres-rustls = "0.14.0"
|
||||
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
arc-swap = "1.9.3"
|
||||
rustfs-tokio-tar = { version = "0.7.1" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.14"
|
||||
tar-framing = "0.0.14"
|
||||
atoi = "3.1.0"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.152.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.119.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.0" }
|
||||
aws-smithy-runtime-api = { version = "1.19.0" }
|
||||
aws-smithy-types = { version = "1.8.1" }
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.126.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.155.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.122.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.1" }
|
||||
aws-smithy-runtime-api = { version = "1.19.1" }
|
||||
aws-smithy-types = { version = "1.9.0" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.0.0"
|
||||
clap = { version = "4.6.7" }
|
||||
@@ -273,12 +272,12 @@ crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.14"
|
||||
enumset = "1.1.15"
|
||||
faster-hex = "1.0.0"
|
||||
flate2 = "1.1.10"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.20.0"
|
||||
google-cloud-auth = "1.17.0"
|
||||
google-cloud-storage = "1.21.0"
|
||||
google-cloud-auth = "1.18.0"
|
||||
hashbrown = { version = "0.17.1" }
|
||||
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
|
||||
# every authenticator app expects). Already in the graph transitively.
|
||||
@@ -288,11 +287,11 @@ highway = { version = "1.3.0" }
|
||||
hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.1"
|
||||
libc = "0.2.190"
|
||||
libc = "0.2.191"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.11"
|
||||
memmap2 = "0.9.12"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
@@ -318,7 +317,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
redis = { version = "1.8.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -328,7 +327,7 @@ s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fc
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.3" }
|
||||
smallvec = { version = "1.16.4" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
@@ -347,7 +346,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.28.0" }
|
||||
uuid = { version = "1.29.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -355,13 +354,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.0.2"
|
||||
zstd = "0.14.2"
|
||||
zip = "9.1.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.3", default-features = false }
|
||||
dial9 = { version = "0.5.5", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -369,12 +368,12 @@ opentelemetry-proto = { version = "0.33.1", default-features = false, features =
|
||||
opentelemetry_sdk = { version = "0.33.1" }
|
||||
opentelemetry-semantic-conventions = { version = "0.33.1" }
|
||||
opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.1" }
|
||||
pyroscope = { version = "2.1.5" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
@@ -385,7 +384,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.6" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.28.6", default-features = false }
|
||||
hotpath = { version = "0.28.7", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -422,8 +421,3 @@ inherits = "release"
|
||||
inherits = "release"
|
||||
debug = true
|
||||
strip = "none"
|
||||
|
||||
# Remove this patch when Pyroscope releases support for object 0.40 or later.
|
||||
# object 0.39 requires ruzstd 0.8, affected by RUSTSEC-2026-0344.
|
||||
[patch.crates-io]
|
||||
pyroscope = { path = "third-party/pyroscope" }
|
||||
|
||||
+14
-1
@@ -18,6 +18,13 @@ ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
# Keep the native store safe even if the distribution CA package lags distrust.
|
||||
COPY scripts/check_ca_bundle.sh /tmp/check_ca_bundle.sh
|
||||
RUN sed -i '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' /etc/ca-certificates.conf && \
|
||||
update-ca-certificates --fresh && \
|
||||
sh /tmp/check_ca_bundle.sh /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs && \
|
||||
rm /tmp/check_ca_bundle.sh
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN set -eux; \
|
||||
@@ -98,7 +105,13 @@ RUN apk upgrade --no-cache && \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
# Keep the native store safe even if the distribution CA package lags distrust.
|
||||
COPY scripts/check_ca_bundle.sh /tmp/check_ca_bundle.sh
|
||||
RUN sed -i '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' /etc/ca-certificates.conf && \
|
||||
update-ca-certificates --fresh && \
|
||||
sh /tmp/check_ca_bundle.sh /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs && \
|
||||
rm /tmp/check_ca_bundle.sh
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
@@ -24,6 +24,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Keep the native store safe even if the distribution CA package lags distrust.
|
||||
COPY scripts/check_ca_bundle.sh /tmp/check_ca_bundle.sh
|
||||
RUN sed -i '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' /etc/ca-certificates.conf && \
|
||||
update-ca-certificates --fresh && \
|
||||
sh /tmp/check_ca_bundle.sh /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs && \
|
||||
rm /tmp/check_ca_bundle.sh
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN set -eux; \
|
||||
@@ -103,6 +110,13 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Keep the native store safe even if the distribution CA package lags distrust.
|
||||
COPY scripts/check_ca_bundle.sh /tmp/check_ca_bundle.sh
|
||||
RUN sed -i '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' /etc/ca-certificates.conf && \
|
||||
update-ca-certificates --fresh && \
|
||||
sh /tmp/check_ca_bundle.sh /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs && \
|
||||
rm /tmp/check_ca_bundle.sh
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
@@ -860,10 +860,12 @@ pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> T
|
||||
if status.is_success() {
|
||||
break response;
|
||||
}
|
||||
if status == StatusCode::INTERNAL_SERVER_ERROR
|
||||
&& response.contains("requires a live fleet capability proof")
|
||||
&& Instant::now() < deadline
|
||||
{
|
||||
// The init stage answers 503 + Retry-After while the cluster capability
|
||||
// proof is still missing; older binaries surfaced the same condition as
|
||||
// a 500 carrying the proof marker.
|
||||
let waiting_for_readiness = status == StatusCode::SERVICE_UNAVAILABLE
|
||||
|| (status == StatusCode::INTERNAL_SERVER_ERROR && response.contains("requires a live fleet capability proof"));
|
||||
if waiting_for_readiness && Instant::now() < deadline {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1206,7 +1206,9 @@ mod tests {
|
||||
let interval = interval
|
||||
.filter(|interval| *interval != 0)
|
||||
.ok_or("scanner schedule has no positive interval")?;
|
||||
let next_cycle = cycle_end
|
||||
// A new leader may already be sleeping before completing its first cycle.
|
||||
let schedule_start = if cycle_end == 0 { now } else { cycle_end };
|
||||
let next_cycle = schedule_start
|
||||
.checked_add(interval)
|
||||
.ok_or("scanner schedule timestamp overflows")?;
|
||||
let budget = next_cycle
|
||||
@@ -1223,6 +1225,11 @@ mod tests {
|
||||
Ok(Duration::from_secs(259)),
|
||||
);
|
||||
assert_eq!(scanner_cycle_completion_budget(100, Some(1), 110, false), Ok(Duration::from_secs(60)));
|
||||
// A newly elected leader can advertise a sleep before its first completed cycle.
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_budget(0, Some(289), 1_792_913_211, false),
|
||||
Ok(Duration::from_secs(349)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1243,7 +1250,7 @@ mod tests {
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let mut deadline = Instant::now() + Duration::from_secs(60);
|
||||
let mut schedule_accounted = false;
|
||||
let mut accounted_schedules = HashSet::new();
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
@@ -1290,15 +1297,27 @@ mod tests {
|
||||
return Err("scanner cycle timestamp is in the future".into());
|
||||
}
|
||||
let schedule = &status["cycle_schedule"];
|
||||
if !schedule_accounted && schedule["execution_role"].as_str() == Some("leader") {
|
||||
if schedule["execution_role"].as_str() == Some("leader") {
|
||||
let active = metrics["current_cycle_active"]
|
||||
.as_bool()
|
||||
.ok_or("scanner status is missing its active-cycle flag")?;
|
||||
if active || (cycle_end != 0 && schedule["effective_interval_available"].as_bool() == Some(true)) {
|
||||
let interval = schedule["effective_interval_seconds"].as_u64();
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
schedule_accounted = true;
|
||||
if active || schedule["effective_interval_available"].as_bool() == Some(true) {
|
||||
let interval = if active {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
schedule["effective_interval_seconds"]
|
||||
.as_u64()
|
||||
.ok_or("scanner schedule is missing its effective interval")?,
|
||||
)
|
||||
};
|
||||
// Reconsider a changed leader or retry plan, but never renew
|
||||
// the deadline on repeated observations of the same sleep.
|
||||
if accounted_schedules.insert((node_index, cycle_end, interval)) {
|
||||
let budget = scanner_cycle_completion_budget(cycle_end, interval, now, active)?;
|
||||
let scheduled_deadline = Instant::now().checked_add(budget).ok_or("scanner deadline overflows")?;
|
||||
deadline = deadline.max(scheduled_deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Superseded walks complete all sets while withholding quota authority.
|
||||
@@ -1969,11 +1988,16 @@ mod tests {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if background_enabled || only_admin_heal_is_active(&status) {
|
||||
let admin_active = background_rejoin_heal_evidence
|
||||
|| (status["state"].as_str() == Some("active")
|
||||
&& status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1));
|
||||
if admin_active && (background_enabled || only_admin_heal_is_active(&status)) {
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("non-admin heal did not drain before the interruption deadline: {status}").into());
|
||||
return Err(
|
||||
format!("heal tasks did not reach the required interruption state before the deadline: {status}").into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
@@ -2667,7 +2691,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_two_node_directory_recovery(remove_bucket: bool, scan_mode: u8) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
async fn verify_two_node_directory_recovery(
|
||||
remove_bucket: bool,
|
||||
scan_mode: u8,
|
||||
bucket_scoped: bool,
|
||||
recreate: bool,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
|
||||
// Isolate explicit administrator recovery from scanner, MRF, and disk healing.
|
||||
@@ -2714,27 +2743,13 @@ mod tests {
|
||||
tokio::fs::create_dir(&removed).await?;
|
||||
if remove_bucket {
|
||||
tokio::fs::rename(&target_bucket, removed.join(bucket)).await?;
|
||||
let status = two_node_heal_status(&cluster, None, false, scan_mode).await?;
|
||||
assert_eq!(
|
||||
status["summary"], "stopped",
|
||||
"missing bucket must not look like an empty successful scan: {status}"
|
||||
);
|
||||
assert_eq!(status["outcome"]["execution"]["state"], "aborted");
|
||||
assert_eq!(status["outcome"]["coverage"], "partial");
|
||||
assert_eq!(status["outcome"]["counters"]["processed"], 0);
|
||||
assert!(!target_bucket.exists(), "default discovery must not silently recreate a bucket");
|
||||
let scoped_default = two_node_heal_status(&cluster, Some(bucket), false, scan_mode).await?;
|
||||
assert_eq!(
|
||||
scoped_default["summary"], "stopped",
|
||||
"default namespace quorum protection must remain: {scoped_default}"
|
||||
);
|
||||
} else {
|
||||
for (index, (key, _)) in objects.iter().enumerate() {
|
||||
tokio::fs::rename(target_bucket.join(key), removed.join(index.to_string())).await?;
|
||||
}
|
||||
}
|
||||
let status = two_node_heal_status(&cluster, remove_bucket.then_some(bucket), remove_bucket, scan_mode).await?;
|
||||
assert_eq!(status["summary"], "finished", "explicit recovery should complete: {status}");
|
||||
let status = two_node_heal_status(&cluster, bucket_scoped.then_some(bucket), recreate, scan_mode).await?;
|
||||
assert_eq!(status["summary"], "finished", "administrator recovery should complete: {status}");
|
||||
assert_eq!(status["outcome"]["execution"]["state"], "completed");
|
||||
assert_eq!(status["outcome"]["coverage"], "complete");
|
||||
for ((key, _), before) in objects.iter().zip(&original) {
|
||||
@@ -2773,16 +2788,26 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_bucket_discovery_and_explicit_recreate() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2).await
|
||||
verify_two_node_directory_recovery(true, 2, true, true).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_missing_bucket_default_cluster_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2, false, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_missing_bucket_default_scoped_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2, true, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_object_directory_recovery_normal() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(false, 1).await
|
||||
verify_two_node_directory_recovery(false, 1, false, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_object_directory_recovery_deep() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(false, 2).await
|
||||
verify_two_node_directory_recovery(false, 2, false, false).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3945,7 +3945,15 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
(sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
|
||||
(large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
|
||||
];
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
// Managed-SSE listings may omit ETags. Check the replicated version's body
|
||||
// before deleting it rather than comparing keyless listing fingerprints.
|
||||
for (key, version_id) in &purged {
|
||||
let source_body = get_version_body(&source_client, &source_bucket, key, version_id).await?;
|
||||
let expected_sha256: [u8; 32] = Sha256::digest(&source_body).into();
|
||||
wait_for_replicated_sha256(&target_client, &target_bucket, key, expected_sha256).await?;
|
||||
let target_body = get_version_body(&target_client, &target_bucket, key, version_id).await?;
|
||||
assert_eq!(target_body, source_body, "replicated version body differs for {key}/{version_id}");
|
||||
}
|
||||
|
||||
for (key, version_id) in &purged {
|
||||
source_client
|
||||
@@ -3956,10 +3964,6 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
// The purge state is applied on the source asynchronously after the target
|
||||
// acknowledges the delete; only then does the retained version go away and
|
||||
// the bucket become deletable. A listing that is empty while DeleteBucket
|
||||
@@ -3984,6 +3988,9 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
}
|
||||
}
|
||||
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -425,10 +425,11 @@ pub mod disk {
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction,
|
||||
DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation,
|
||||
DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions,
|
||||
FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
|
||||
RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
DeleteRecoveryRpcReply, DeleteRecoveryState, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
|
||||
@@ -525,6 +525,36 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
|
||||
sys.get_bucket_incarnation_id_from_disk(bucket).await
|
||||
}
|
||||
|
||||
/// Directory loss is recoverable only for a committed, non-retired generation.
|
||||
/// The caller retains the bucket lifecycle fence through the ensuing reconstruction.
|
||||
pub(crate) async fn validate_bucket_reconstruction_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
) -> Result<()> {
|
||||
let api = bucket_metadata_sys_of(ctx)?.read().await.object_store();
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = load_bucket_metadata_parse_with_presence(api.clone(), bucket, true).await?;
|
||||
if !persisted
|
||||
|| !metadata.bucket_incarnation_sidecar
|
||||
|| expected.is_nil()
|
||||
|| metadata.bucket_incarnation_id != expected
|
||||
|| !metadata.bucket_creation_committed
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if crate::bucket::retirement::is_retired(api, bucket, expected).await? {
|
||||
return Err(Error::BucketNotFound(bucket.to_owned()));
|
||||
}
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
|
||||
@@ -65,9 +65,9 @@ use super::replication_target_boundary::{
|
||||
SsecPassthroughGate, TargetClient, VersionIdentityCapability, is_replication_target_offline_error,
|
||||
replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
|
||||
replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_source_etag_is_comparable,
|
||||
replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
|
||||
ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -349,15 +349,22 @@ fn replica_etag_verification_enabled() -> bool {
|
||||
/// target that stores a transformed payload (e.g. undecoded `aws-chunked`
|
||||
/// frames, #6853) returns the ETag of what it actually wrote. Reporting
|
||||
/// COMPLETED over such a replica is silent corruption, so a decidable
|
||||
/// mismatch fails the replication instead. An SSE-C ciphertext passthrough
|
||||
/// transfer is exempt: the wire bytes are ciphertext while the source ETag is
|
||||
/// the plaintext MD5, and that path has its own HEAD-back audit.
|
||||
/// mismatch fails the replication instead. Encrypted sources are exempt: an
|
||||
/// SSE-C ciphertext passthrough sends ciphertext while the source ETag is the
|
||||
/// plaintext MD5 (that path has its own HEAD-back audit), and a managed-SSE
|
||||
/// (SSE-S3 / SSE-KMS) source stores a keyed fingerprint, or a legacy plaintext
|
||||
/// MD5, that the ETag of the target's re-encrypted replica can neither confirm
|
||||
/// nor refute. The checksum header on the replication PUT stays the integrity
|
||||
/// check for those objects.
|
||||
fn verify_single_part_replica(
|
||||
object_info: &ObjectInfo,
|
||||
response: &RemotePutObjectResponse,
|
||||
ciphertext_passthrough: bool,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
if ciphertext_passthrough || !replica_etag_verification_enabled() {
|
||||
if ciphertext_passthrough
|
||||
|| !replication_source_etag_is_comparable(&object_info.user_defined)
|
||||
|| !replica_etag_verification_enabled()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) {
|
||||
@@ -5870,6 +5877,57 @@ mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError;
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_exempts_managed_sse_sources() {
|
||||
// A new encrypted write stores a keyed fingerprint; a target that
|
||||
// computes its own ETag answers with the MD5 of the plaintext it
|
||||
// received. Both are 32 hex characters, so without the exemption the
|
||||
// audit would fail every SSE-S3 / SSE-KMS replica (rustfs/backlog).
|
||||
let fingerprint = "00112233445566778899aabbccddeeff";
|
||||
let replica_md5 = "ffeeddccbbaa99887766554433221100";
|
||||
let response = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some(replica_md5.to_string()),
|
||||
};
|
||||
for sse in ["AES256", "aws:kms"] {
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "encrypted".to_string(),
|
||||
etag: Some(fingerprint.to_string()),
|
||||
user_defined: HashMap::from([("x-amz-server-side-encryption".to_string(), sse.to_string())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
verify_single_part_replica(&object_info, &response, false)
|
||||
.unwrap_or_else(|err| panic!("{sse}: a managed-SSE source must not be audited against the replica MD5: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_still_fails_plaintext_mismatch() {
|
||||
if !replica_etag_verification_enabled() {
|
||||
return;
|
||||
}
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "plain".to_string(),
|
||||
etag: Some("00112233445566778899aabbccddeeff".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mismatch = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some("ffeeddccbbaa99887766554433221100".to_string()),
|
||||
};
|
||||
let err = verify_single_part_replica(&object_info, &mismatch, false)
|
||||
.expect_err("a plaintext source whose replica MD5 differs must fail the replication");
|
||||
assert_eq!(err.to_string(), REPLICA_ETAG_MISMATCH_ERROR);
|
||||
|
||||
let matching = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: object_info.etag.clone(),
|
||||
};
|
||||
verify_single_part_replica(&object_info, &matching, false).expect("matching plaintext MD5s pass the audit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_preserves_legacy_plain_part_ranges() {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
|
||||
@@ -157,6 +157,19 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// Whether the stored ETag of a replication source can be compared with the
|
||||
/// ETag a target returns for the bytes it received. Only an unencrypted source
|
||||
/// stores the MD5 of the bytes that go on the wire. A managed-SSE (SSE-S3 /
|
||||
/// SSE-KMS) or SSE-C source stores a keyed fingerprint (new writes) or a legacy
|
||||
/// plaintext MD5, and the target re-encrypts what it receives, so neither
|
||||
/// equality nor inequality of the two ETags proves anything about the replica.
|
||||
pub(crate) fn replication_source_etag_is_comparable(user_defined: &HashMap<String, String>) -> bool {
|
||||
matches!(
|
||||
classify_replication_source_encryption(user_defined),
|
||||
ReplicationSourceEncryption::Plaintext
|
||||
)
|
||||
}
|
||||
|
||||
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||
/// customer-algorithm header and let the crate-owned policy decide.
|
||||
@@ -578,6 +591,22 @@ mod tests {
|
||||
use time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn replication_source_etag_is_comparable_only_for_plaintext_sources() {
|
||||
assert!(replication_source_etag_is_comparable(&HashMap::new()));
|
||||
for (key, value) in [
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "AES256"),
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "aws:kms"),
|
||||
(SSEC_ALGORITHM_HEADER, "AES256"),
|
||||
] {
|
||||
let metadata = HashMap::from([(key.to_string(), value.to_string())]);
|
||||
assert!(
|
||||
!replication_source_etag_is_comparable(&metadata),
|
||||
"{key}={value} must not expose its stored ETag to the replica audit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the
|
||||
/// SSE intent as lowercase `x-amz-server-side-encryption`
|
||||
/// (`encryption_material_to_metadata`; header-derived keys come from a
|
||||
|
||||
@@ -3691,6 +3691,55 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_worker_read_lock_error(err: rustfs_lock::LockError, object: &str) -> Error {
|
||||
match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object: object.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::Lock(other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only admission to an already durable rebalance run. Keeping this
|
||||
/// separate from the activation writer prevents admission from authorizing a
|
||||
/// pool or rebalance metadata write under shared namespace locks.
|
||||
pub(crate) struct PoolRebalanceWorkerFence {
|
||||
pool_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
}
|
||||
|
||||
impl PoolRebalanceWorkerFence {
|
||||
pub(crate) fn set_fleet_proof(
|
||||
&mut self,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
) {
|
||||
self.fleet_proof = fleet_proof;
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> Result<()> {
|
||||
if self.pool_meta_guard.is_lock_lost()
|
||||
|| self.pool_meta_guard.is_released()
|
||||
|| self.rebalance_meta_guard.is_lock_lost()
|
||||
|| self.rebalance_meta_guard.is_released()
|
||||
{
|
||||
return Err(Error::other("worker admission lock lost before rebalance activation"));
|
||||
}
|
||||
if self
|
||||
.fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
|
||||
{
|
||||
return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct PoolRebalanceActivationFence {
|
||||
pool_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
rebalance_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
@@ -3762,6 +3811,35 @@ tokio::task_local! {
|
||||
pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_worker_locks<S>(pool: Arc<S>) -> Result<PoolRebalanceWorkerFence>
|
||||
where
|
||||
S: crate::storage_api_contracts::namespace::NamespaceLocking<
|
||||
Error = Error,
|
||||
NamespaceLock = rustfs_lock::NamespaceLockWrapper,
|
||||
>,
|
||||
{
|
||||
// Match activation and entry admission: rebalance.bin -> pool.bin. Joining
|
||||
// the same durable run must coexist with entries retaining their run fence.
|
||||
let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
#[cfg(test)]
|
||||
let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
|
||||
let rebalance_meta_guard = rebalance_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, REBAL_META_NAME))?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, POOL_META_NAME))?;
|
||||
|
||||
Ok(PoolRebalanceWorkerFence {
|
||||
pool_meta_guard,
|
||||
rebalance_meta_guard,
|
||||
fleet_proof: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
|
||||
pool: Arc<S>,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
@@ -12430,6 +12508,18 @@ impl ECStore {
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_worker_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
worker_fence: &PoolRebalanceWorkerFence,
|
||||
operation: &str,
|
||||
) -> Result<PoolMeta> {
|
||||
worker_fence.ensure_held()?;
|
||||
let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
|
||||
@@ -2426,7 +2426,7 @@ pub(crate) fn read_destination_file_with_commit_guard(
|
||||
let Some(parent) = directories.destination.get() else {
|
||||
return Ok(None);
|
||||
};
|
||||
return match open_linux_relative(&parent.file, Path::new(name), OFlags::RDONLY, Mode::empty()) {
|
||||
match open_linux_relative(&parent.file, Path::new(name), OFlags::RDONLY, Mode::empty()) {
|
||||
Ok(mut file) => {
|
||||
let mut data = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut data)?;
|
||||
@@ -2434,7 +2434,7 @@ pub(crate) fn read_destination_file_with_commit_guard(
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
@@ -2504,9 +2504,9 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
{
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
return commit_guard
|
||||
commit_guard
|
||||
.directories
|
||||
.rename(&src_file_path, &dst_file_path, Some(&prepared_source));
|
||||
.rename(&src_file_path, &dst_file_path, Some(&prepared_source))
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
@@ -2542,7 +2542,7 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
let destination = dst_file_path.as_ref().to_path_buf();
|
||||
let guard = commit_guard.clone();
|
||||
let _ = (base_dir, publication_root);
|
||||
return run_blocking_namespace_operation(lease, move || {
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
guard.directories.prepare_destination()?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_rename_destination(&source, &destination);
|
||||
@@ -2555,7 +2555,7 @@ pub(crate) async fn rename_all_with_commit_guard(
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from);
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
@@ -3890,7 +3890,7 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
return Ok(RenameCommitGuard {
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: publication_root.clone(),
|
||||
source_path: source_parent.to_path_buf(),
|
||||
@@ -3899,7 +3899,7 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
destination,
|
||||
destination_base: destination_base.to_path_buf(),
|
||||
}),
|
||||
});
|
||||
})
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
|
||||
@@ -1006,6 +1006,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive.
|
||||
pub bucket_lifecycle_lock_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
pub replication_request: bool,
|
||||
/// True when the inbound request carried the
|
||||
/// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
|
||||
@@ -1165,6 +1168,7 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("delete_mutation_owner", &self.delete_mutation_owner.is_some())
|
||||
.field("namespace_lock_fence", &self.namespace_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_owner", &self.bucket_lifecycle_lock_owner.is_some())
|
||||
.field("replication_request", &self.replication_request)
|
||||
.field("proxy_request", &self.proxy_request)
|
||||
.field("proxy_header_set", &self.proxy_header_set)
|
||||
|
||||
@@ -18,8 +18,8 @@ use super::{
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
|
||||
pool_meta_has_active_decommission,
|
||||
PoolMeta, PoolRebalanceActivationFence, PoolRebalanceWorkerFence, acquire_pool_activation_fleet_proof,
|
||||
acquire_pool_rebalance_activation_locks, acquire_pool_rebalance_worker_locks, pool_meta_has_active_decommission,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -77,10 +77,33 @@ fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()>
|
||||
}
|
||||
|
||||
pub(super) enum RebalanceWorkerActivationFence {
|
||||
Ready(Box<PoolRebalanceActivationFence>),
|
||||
Ready(Box<PoolRebalanceWorkerFence>),
|
||||
NotStartedTerminal,
|
||||
}
|
||||
|
||||
async fn persisted_rebalance_requires_worker_activation<S>(pool: Arc<S>, expected_id: &str) -> Result<bool>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
Ok(crate::services::rebalance::rebalance_requires_worker_activation(&persisted))
|
||||
}
|
||||
|
||||
pub(super) struct RebalanceRunGuard {
|
||||
_local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
|
||||
persisted_guard: rustfs_lock::NamespaceLockGuard,
|
||||
@@ -577,48 +600,58 @@ impl ECStore {
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
||||
// shared fences admit a peer without waiting for the run's own entries.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
// Classify the durable rebalance record while holding both namespace
|
||||
// fences. A terminal record is a no-op and must not depend on the
|
||||
// notification subsystem having published a fleet proof yet.
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
|
||||
.load_runtime_pool_meta_under_worker_fence(&mut pool_meta_guard, &worker_fence, "rebalance worker activation")
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
|
||||
activation_fence.ensure_held()?;
|
||||
if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
|
||||
}
|
||||
|
||||
// Active worker admission still requires the fail-closed fleet proof.
|
||||
// Attach it immediately before the final fence validation so expiry or
|
||||
// topology changes are checked again at every later commit boundary.
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
worker_fence.set_fleet_proof(fleet_proof);
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(worker_fence)))
|
||||
}
|
||||
|
||||
pub(super) async fn fence_rebalance_worker_metadata_update<S>(
|
||||
&self,
|
||||
pool: Arc<S>,
|
||||
expected_id: &str,
|
||||
) -> Result<Option<Box<PoolRebalanceActivationFence>>>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// The admission read fences must already be dropped. Reacquire in the
|
||||
// original order and revalidate every durable boundary across that gap.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(
|
||||
&mut pool_meta_guard,
|
||||
&activation_fence,
|
||||
"rebalance worker metadata update",
|
||||
)
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
activation_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
activation_fence.set_fleet_proof(fleet_proof);
|
||||
activation_fence.ensure_held()?;
|
||||
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
|
||||
Ok(Some(Box::new(activation_fence)))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
@@ -1816,6 +1849,107 @@ mod tests {
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_init_missing_fleet_proof_preserves_metadata_and_rechecks_competing_run() {
|
||||
let previous_id = "completed-before-capability-refresh";
|
||||
let terminal = RebalanceMeta {
|
||||
id: previous_id.to_string(),
|
||||
stopped_at: Some(OffsetDateTime::now_utc()),
|
||||
pool_stats: vec![RebalanceStats::default(); 2],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, competing_store) = crate::services::rebalance::test_two_pool_stores(Some(terminal.clone())).await;
|
||||
let disk_stats = vec![
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 0,
|
||||
},
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 100,
|
||||
},
|
||||
];
|
||||
let initial_epoch = store.ctx.data_movement_operation_epoch();
|
||||
let proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats.clone());
|
||||
let err = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("initialization must fail closed while the live proof is missing");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("pool activation requires a live fleet capability proof")
|
||||
);
|
||||
assert!(
|
||||
crate::core::pools::is_pool_activation_fleet_proof_error(&err),
|
||||
"the admin handler retries init only on this error shape: {err}"
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch);
|
||||
{
|
||||
let local = store.rebalance_meta.read().await;
|
||||
let local = local.as_ref().expect("failed initialization must preserve the previous run");
|
||||
assert_eq!(local.id, previous_id);
|
||||
assert_eq!(local.stopped_at, terminal.stopped_at);
|
||||
assert!(local.cancel.is_none(), "failed initialization must not start a worker");
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("failed initialization must leave durable metadata readable");
|
||||
assert_eq!(persisted.id, previous_id);
|
||||
assert_eq!(persisted.stopped_at, terminal.stopped_at);
|
||||
|
||||
drop(proof_guard);
|
||||
set_rebalance_disk_stats_override_for_test(competing_store.id, disk_stats.clone());
|
||||
let winning_id = competing_store
|
||||
.init_rebalance_start(vec!["winning-request".to_string()])
|
||||
.await
|
||||
.expect("a ready request must be able to initialize one run after the failed attempt");
|
||||
assert_ne!(winning_id, previous_id);
|
||||
|
||||
// The waiting request still holds its old local snapshot. Its retry must
|
||||
// recheck durable state under the activation locks before publishing.
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats);
|
||||
let remote_conflict = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("the waiting request must not replace a durable competing run");
|
||||
assert!(
|
||||
matches!(
|
||||
crate::data_movement::data_movement_stage_source(&remote_conflict),
|
||||
Some(Error::RebalanceAlreadyRunning)
|
||||
),
|
||||
"the save-stage error must retain the typed competing-run conflict: {remote_conflict:?}"
|
||||
);
|
||||
let local_conflict = competing_store
|
||||
.init_rebalance_start(vec!["another-request".to_string()])
|
||||
.await
|
||||
.expect_err("a local retry must not replace its active run");
|
||||
assert!(
|
||||
matches!(local_conflict, Error::RebalanceAlreadyRunning),
|
||||
"the local admission check must report the competing-run conflict: {local_conflict:?}"
|
||||
);
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing run must remain durable after both rejected retries");
|
||||
assert_eq!(persisted.id, winning_id);
|
||||
assert_eq!(persisted.pool_stats[0].buckets, vec!["winning-request".to_string()]);
|
||||
assert_eq!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("old local run must survive")
|
||||
.id,
|
||||
previous_id
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
|
||||
@@ -1874,6 +2008,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_worker_write_upgrade_revalidates_durable_state(change: &str) {
|
||||
let rebalance_id = "rebalance-worker-write-upgrade";
|
||||
let active = RebalanceMeta {
|
||||
id: rebalance_id.to_string(),
|
||||
percent_free_goal: 0.5,
|
||||
pool_stats: vec![
|
||||
RebalanceStats {
|
||||
participating: true,
|
||||
init_capacity: 100,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStats::default(),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, peer) =
|
||||
crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(active.clone())).await;
|
||||
let arrived = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let worker = Arc::clone(&store);
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
tasks.spawn(
|
||||
super::super::runtime::REBALANCE_WORKER_WRITE_UPGRADE_BARRIER
|
||||
.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
|
||||
worker.start_rebalance_for_id(rebalance_id).await
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), arrived.notified())
|
||||
.await
|
||||
.expect("empty-queue activation must release its read fences before requesting write authority");
|
||||
|
||||
let mut changed = active.clone();
|
||||
match change {
|
||||
"stopped" => {
|
||||
changed.stopped_at = Some(OffsetDateTime::now_utc());
|
||||
changed.pool_stats[0].info.status = RebalStatus::Stopped;
|
||||
}
|
||||
"replaced" => changed.id = "replacement-rebalance-run".to_string(),
|
||||
"decommission" => {
|
||||
let mut pool_meta = peer.pool_meta.read().await.clone();
|
||||
pool_meta.pools[0].decommission = Some(crate::core::pools::PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
pool_meta
|
||||
.save(peer.pools.clone())
|
||||
.await
|
||||
.expect("a competing pool metadata writer must acquire the released pool fence");
|
||||
}
|
||||
_ => panic!("unknown durable state transition"),
|
||||
}
|
||||
changed
|
||||
.save_with_opts(
|
||||
peer.pools[0].clone(),
|
||||
ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a competing run writer must acquire the released rebalance fence");
|
||||
release.notify_one();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(30), tasks.join_next())
|
||||
.await
|
||||
.expect("activation must finish after the durable transition")
|
||||
.expect("activation task must be tracked")
|
||||
.expect("activation must not panic");
|
||||
match change {
|
||||
"stopped" => result.expect("a durably stopped run must become an activation no-op"),
|
||||
"replaced" => assert!(
|
||||
result
|
||||
.expect_err("a replaced run must reject stale activation")
|
||||
.to_string()
|
||||
.contains("expected"),
|
||||
),
|
||||
"decommission" => assert!(matches!(result, Err(Error::DecommissionAlreadyRunning))),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(peer.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing durable state must remain readable");
|
||||
assert_eq!(persisted.id, changed.id, "activation must not overwrite the run identity");
|
||||
assert_eq!(persisted.stopped_at, changed.stopped_at, "activation must not overwrite the stop marker");
|
||||
assert_eq!(persisted.pool_stats[0].info.status, changed.pool_stats[0].info.status);
|
||||
assert!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("local metadata must remain present")
|
||||
.cancel
|
||||
.is_none(),
|
||||
"rejected activation must not install a worker token",
|
||||
);
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_stop() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("stopped").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_run_identity() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("replaced").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_decommission() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("decommission").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
|
||||
|
||||
@@ -1440,7 +1440,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
|
||||
async fn real_rebalance_peer_activates_while_entry_retains_run_fence() {
|
||||
const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
|
||||
let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
|
||||
active_rebalance_meta(REBALANCE_ID),
|
||||
@@ -1456,7 +1456,7 @@ mod tests {
|
||||
let bucket = crate::disk::RUSTFS_META_BUCKET;
|
||||
let object = "rebalance-peer-activation-object";
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
|
||||
let payload = b"peer activation must not wait for this entry to drain".repeat(1024);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let opts = ObjectOptions {
|
||||
@@ -1516,7 +1516,12 @@ mod tests {
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
|
||||
.await
|
||||
.expect("peer activation must attempt the persisted rebalance write fence");
|
||||
.expect("peer activation must attempt the persisted rebalance fence");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), activation_result)
|
||||
.await
|
||||
.expect("peer activation must finish while the entry remains paused")
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("joining the same run must not require its active entry to release the run fence");
|
||||
release.notify_one();
|
||||
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
@@ -1531,10 +1536,6 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect("entry and peer activation must both make progress");
|
||||
activation_result
|
||||
.await
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("peer activation must not time out behind the entry it blocks");
|
||||
|
||||
let mut reader = target_set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
|
||||
@@ -27,6 +27,11 @@ use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(super) static REBALANCE_WORKER_WRITE_UPGRADE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
|
||||
}
|
||||
|
||||
pub(super) fn should_fail_repeated_rebalance_bucket_defer(
|
||||
deferred_buckets: &mut HashSet<String>,
|
||||
bucket: &str,
|
||||
@@ -213,6 +218,22 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if must_persist {
|
||||
// Shared admission grants no write authority. Release it before
|
||||
// taking the exclusive metadata fences, then recheck stop/run/pool
|
||||
// state before publishing the locally staged terminal candidate.
|
||||
drop(activation_fence);
|
||||
#[cfg(test)]
|
||||
if let Ok((arrived, release)) = REBALANCE_WORKER_WRITE_UPGRADE_BARRIER.try_with(Clone::clone) {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
let Some(activation_fence) = self
|
||||
.fence_rebalance_worker_metadata_update(pool.clone(), expected_id.as_ref())
|
||||
.await?
|
||||
else {
|
||||
cancel_tx.cancel();
|
||||
return Ok(false);
|
||||
};
|
||||
let save_result = resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_under_activation_fence(
|
||||
pool,
|
||||
@@ -242,13 +263,14 @@ impl ECStore {
|
||||
cancel_tx.cancel();
|
||||
return Err(err);
|
||||
}
|
||||
} else {
|
||||
if let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
}
|
||||
if !must_persist && let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
|
||||
if activation_outcome != RebalanceLocalActivationOutcome::Started {
|
||||
return Ok(must_persist);
|
||||
|
||||
@@ -31,6 +31,7 @@ use crate::disk::DiskAPI;
|
||||
pub(crate) enum BucketInfoQuorum {
|
||||
Read,
|
||||
Write,
|
||||
HealCandidates,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
@@ -104,7 +105,9 @@ impl SetDisks {
|
||||
// class. A namespace read must intersect that majority; object
|
||||
// readers still enforce the persisted layout's data-shard quorum.
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
|
||||
reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1)
|
||||
}
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
@@ -126,6 +129,29 @@ impl SetDisks {
|
||||
self.list_bucket_with_quorum(BucketInfoQuorum::Write).await
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_needs_reconstruction(&self, bucket: &str) -> Result<bool> {
|
||||
let results = join_all(self.disk_inventory().await.into_iter().map(|disk| async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let mut present = false;
|
||||
let mut missing = false;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => present = true,
|
||||
Err(DiskError::VolumeNotFound) => missing = true,
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if !present {
|
||||
return Err(Error::VolumeNotFound);
|
||||
}
|
||||
Ok(missing)
|
||||
}
|
||||
|
||||
pub(crate) async fn list_bucket_with_quorum(&self, quorum: BucketInfoQuorum) -> Result<(Vec<BucketInfo>, bool)> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
@@ -134,6 +160,7 @@ impl SetDisks {
|
||||
let required = match quorum {
|
||||
BucketInfoQuorum::Read => disks.len().div_ceil(2).max(1),
|
||||
BucketInfoQuorum::Write => write_quorum,
|
||||
BucketInfoQuorum::HealCandidates => disks.len().max(1),
|
||||
};
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
@@ -165,7 +192,9 @@ impl SetDisks {
|
||||
|
||||
let error = match quorum {
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
|
||||
reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required)
|
||||
}
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
@@ -190,10 +219,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
|
||||
let minimum_presence = if matches!(quorum, BucketInfoQuorum::HealCandidates) {
|
||||
// Candidate presence is not mutation authority. Every disk must answer,
|
||||
// and the heal owner validates the committed generation before recreation.
|
||||
1
|
||||
} else {
|
||||
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
|
||||
required
|
||||
};
|
||||
let mut buckets = counts
|
||||
.into_values()
|
||||
.filter_map(|(count, bucket)| (count >= required).then_some(bucket))
|
||||
.filter_map(|(count, bucket)| (count >= minimum_presence).then_some(bucket))
|
||||
.collect::<Vec<_>>();
|
||||
buckets.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
Ok((buckets, topology_complete))
|
||||
|
||||
@@ -63,7 +63,6 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::disk::DiskAPI;
|
||||
#[cfg(test)]
|
||||
use crate::disk::DiskOption;
|
||||
#[cfg(test)]
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
#[cfg(test)]
|
||||
use crate::disk::new_disk;
|
||||
@@ -231,6 +230,60 @@ pub(crate) struct StaleMultipartCleanupGuard {
|
||||
lock_guard: ObjectLockDiagGuard,
|
||||
}
|
||||
|
||||
/// Exact-key recovery is admitted only after every disk proves that the upload
|
||||
/// metadata is absent. Keep the checked disk generation and upload lock through
|
||||
/// deletion; a quorum miss is not proof that an upload is an orphan.
|
||||
/// Final object data, completion replay identity, and old-data cleanup receipts
|
||||
/// live under the destination bucket/object, never this staging path. This also
|
||||
/// permits removing an exact orphan from a previous bucket incarnation without
|
||||
/// claiming that its UploadID belongs to the current incarnation.
|
||||
pub(crate) struct OrphanMultipartCleanupGuard {
|
||||
set: Arc<SetDisks>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
upload_path: String,
|
||||
lock_guard: Arc<ObjectLockDiagGuard>,
|
||||
exists: bool,
|
||||
}
|
||||
|
||||
impl OrphanMultipartCleanupGuard {
|
||||
pub(crate) async fn delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<bool> {
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
if !self.exists {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let results = join_all(self.disks.iter().map(|disk| async {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
match disk
|
||||
.delete(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&self.upload_path,
|
||||
crate::disk::DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => Ok(()),
|
||||
result => result,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
self.set.record_capacity_scope_if_needed(None, &self.disks);
|
||||
let errors = results.into_iter().map(|result| result.err()).collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, self.disks.len()) {
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DataMovementMultipartAbortGuard {
|
||||
upload_path: String,
|
||||
write_quorum: Option<usize>,
|
||||
@@ -557,6 +610,34 @@ fn multipart_bucket_incarnation_matches(metadata: &HashMap<String, String>, expe
|
||||
matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected)
|
||||
}
|
||||
|
||||
fn orphan_multipart_upload_id_is_canonical(upload_id: &str) -> bool {
|
||||
let Ok(decoded) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(decoded) = std::str::from_utf8(&decoded) else {
|
||||
return false;
|
||||
};
|
||||
let Some((_, raw)) = decoded.split_once('.') else {
|
||||
return false;
|
||||
};
|
||||
let Some((uuid, timestamp)) = raw.split_once('x') else {
|
||||
return false;
|
||||
};
|
||||
let Ok(id) = Uuid::parse_str(uuid) else {
|
||||
return false;
|
||||
};
|
||||
if id.is_nil() || id.to_string() != uuid {
|
||||
return false;
|
||||
}
|
||||
let Ok(timestamp_ns) = timestamp.parse::<i128>() else {
|
||||
return false;
|
||||
};
|
||||
if timestamp_ns.to_string() != timestamp {
|
||||
return false;
|
||||
}
|
||||
OffsetDateTime::from_unix_timestamp_nanos(timestamp_ns).is_ok()
|
||||
}
|
||||
|
||||
fn validate_multipart_bucket_incarnation(
|
||||
metadata: &HashMap<String, String>,
|
||||
bucket: &str,
|
||||
@@ -1246,6 +1327,114 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn lock_orphan_multipart_cleanup(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: &ObjectOptions,
|
||||
held_guards: &[OrphanMultipartCleanupGuard],
|
||||
) -> Result<OrphanMultipartCleanupGuard> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if opts.data_movement
|
||||
|| opts.no_lock
|
||||
|| self.local_lock_manager.is_disabled()
|
||||
|| !orphan_multipart_upload_id_is_canonical(upload_id)
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
// This path is derived from the caller's exact bucket/key and a canonical
|
||||
// UUID/timestamp upload identity, never from the shared MPU directory list.
|
||||
let upload_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
// Pool labels do not partition the raw ObjectKey sent to lock clients.
|
||||
// Reuse the actual owner only for this exact path and proven backend;
|
||||
// independent distributed domains still need their own guards.
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
let mut lock_guard = None;
|
||||
for held in held_guards {
|
||||
if held.upload_path != upload_path {
|
||||
continue;
|
||||
}
|
||||
let shared_domain = match (distributed, held.set.ctx.is_dist_erasure().await) {
|
||||
(false, false) => self.shares_namespace_lock_domain(&held.set).await,
|
||||
(true, true) => {
|
||||
let same = super::super::same_distributed_lock_domain(&self.lockers, &held.set.lockers);
|
||||
if !same
|
||||
&& self
|
||||
.lockers
|
||||
.iter()
|
||||
.any(|client| held.set.lockers.iter().any(|held_client| Arc::ptr_eq(client, held_client)))
|
||||
{
|
||||
// A guard for only part of another quorum is not proof
|
||||
// of ownership there, and acquiring it again may wait
|
||||
// on ourselves. Preserve all staging in this topology.
|
||||
return Err(Error::other("orphan multipart cleanup cannot cover partially overlapping lock domains"));
|
||||
}
|
||||
same
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if shared_domain {
|
||||
lock_guard = Some(Arc::clone(&held.lock_guard));
|
||||
break;
|
||||
}
|
||||
}
|
||||
let lock_guard = match lock_guard {
|
||||
Some(guard) => guard,
|
||||
None => Arc::new(
|
||||
self.acquire_write_lock_diag("orphan_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await?,
|
||||
),
|
||||
};
|
||||
fence_commit_on_lock_loss(Some(&lock_guard), "orphan_multipart_cleanup", &upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.len() != self.set_drive_count || disks.is_empty() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
// Raw full fan-out cannot quorum-short-circuit or interpret an empty
|
||||
// version list as absence of the physical metadata file.
|
||||
let (_, errors) = Self::read_all_raw_file_info(&disks, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
if errors.len() != disks.len() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for error in errors {
|
||||
match error {
|
||||
Some(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
// Even one valid metadata copy may still be recoverable. Never
|
||||
// convert a partial, corrupt, or unreachable upload into debris.
|
||||
None => return Err(invalid_upload()),
|
||||
Some(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
let mut exists = false;
|
||||
for disk in &disks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await {
|
||||
Ok(entries) => {
|
||||
// Raw metadata reads report a directory named xl.meta as
|
||||
// FileNotFound. Its physical presence is unknown/corrupt
|
||||
// metadata, not proof that this staging can be deleted.
|
||||
if entries.iter().any(|entry| entry.trim_end_matches('/') == STORAGE_FORMAT_FILE) {
|
||||
return Err(to_object_err(DiskError::FileCorrupt.into(), vec![bucket, object, upload_id]));
|
||||
}
|
||||
exists = true;
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
Err(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
Ok(OrphanMultipartCleanupGuard {
|
||||
set: self.clone(),
|
||||
disks,
|
||||
upload_path,
|
||||
lock_guard,
|
||||
exists,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1678,6 +1867,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
drop(admission_guard);
|
||||
}
|
||||
|
||||
let mut commit_owns_cleanup = false;
|
||||
let result: Result<PartInfo> = async {
|
||||
let erasure =
|
||||
Arc::new(coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
|
||||
@@ -1881,120 +2071,171 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?;
|
||||
(Some(upload_guard), Some(part_guard))
|
||||
};
|
||||
let (commit_fi, _) = self
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&self.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = self
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[self.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = self
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
// Only guards owned by this operation may outlive its caller. Internal
|
||||
// data-movement and borrowed-lock callers retain their synchronous
|
||||
// lifetime; a fence by itself cannot transfer the outer lock lease.
|
||||
let detach_commit_owner = _upload_commit_guard.is_some()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_upload_id = upload_id.to_owned();
|
||||
let commit_upload_id_path = upload_id_path.clone();
|
||||
let commit_tmp_part = tmp_part.clone();
|
||||
let commit_opts = opts.clone();
|
||||
commit_owns_cleanup = true;
|
||||
// The body is already staged. Keep every commit guard through the
|
||||
// prepare, publish, settlement/rollback and temporary cleanup I/O,
|
||||
// even when the HTTP request stops awaiting this task.
|
||||
let commit = async move {
|
||||
let _upload_commit_guard = _upload_commit_guard;
|
||||
let _part_commit_guard = _part_commit_guard;
|
||||
let _capped_staging_guard = _capped_staging_guard;
|
||||
let bucket = commit_bucket.as_str();
|
||||
let object = commit_object.as_str();
|
||||
let upload_id = commit_upload_id.as_str();
|
||||
let upload_id_path = commit_upload_id_path;
|
||||
let opts = &commit_opts;
|
||||
let result: Result<PartInfo> = async {
|
||||
let (commit_fi, _) = commit_set
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&commit_set.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = commit_set
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[commit_set.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = commit_set
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
drop(_part_commit_guard);
|
||||
drop(_upload_commit_guard);
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
if result.is_err()
|
||||
&& let Err(err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_part).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
result = "cleanup_failed",
|
||||
bucket,
|
||||
object,
|
||||
error = ?err,
|
||||
"failed to clean multipart commit staging"
|
||||
);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
if detach_commit_owner {
|
||||
tokio::spawn(commit)
|
||||
.await
|
||||
.map_err(|err| Error::other_with_context("put_object_part commit task failed", err))?
|
||||
} else {
|
||||
commit.await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err()
|
||||
if !commit_owns_cleanup
|
||||
&& result.is_err()
|
||||
&& let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_part).await
|
||||
{
|
||||
warn!(tmp_part = %tmp_part, error = ?err, "failed to cleanup multipart temporary data");
|
||||
@@ -4233,6 +4474,300 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_multipart_identity_requires_canonical_upload_id() {
|
||||
let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
let id = Uuid::new_v4();
|
||||
let timestamp = (created + time::Duration::seconds(1)).unix_timestamp_nanos();
|
||||
let encode = |raw: &str| runtime_sources::deployment_upload_id(raw);
|
||||
let upload_id = encode(&format!("{id}x{timestamp}"));
|
||||
assert!(orphan_multipart_upload_id_is_canonical(&upload_id));
|
||||
for raw in [
|
||||
format!("{}x{timestamp}", Uuid::nil()),
|
||||
format!("{}x{timestamp}", id.simple()),
|
||||
format!("{id}x0{timestamp}"),
|
||||
format!("{id}x+{timestamp}"),
|
||||
format!("{id}x{timestamp}/../another-upload"),
|
||||
format!("../{id}x{timestamp}"),
|
||||
format!("{id}x{}", i128::MAX),
|
||||
] {
|
||||
assert!(
|
||||
!orphan_multipart_upload_id_is_canonical(&encode(&raw)),
|
||||
"noncanonical upload identity must be rejected: {raw}"
|
||||
);
|
||||
}
|
||||
assert!(!orphan_multipart_upload_id_is_canonical("not-base64"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_requires_every_disk_to_prove_metadata_absence() {
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-proof";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
for disk in &disks[..3] {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove majority metadata");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one surviving metadata copy must block orphan recovery"
|
||||
);
|
||||
disks[3]
|
||||
.write_all(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Bytes::from_static(b"corrupt metadata"))
|
||||
.await
|
||||
.expect("corrupt final metadata copy");
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt metadata must not be treated as absence"
|
||||
);
|
||||
disks[3]
|
||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove final metadata copy");
|
||||
set.disks.write().await[3] = None;
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"an unreachable disk must not be hidden by absence quorum"
|
||||
);
|
||||
set.disks.write().await[3] = Some(disks[3].clone());
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("all disks now prove metadata absence");
|
||||
assert!(guard.delete(bucket, object, &opts).await.expect("recover exact staging"));
|
||||
drop(guard);
|
||||
for disk in &disks {
|
||||
assert!(matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
|
||||
));
|
||||
}
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("already absent staging remains provably absent");
|
||||
assert!(
|
||||
!guard
|
||||
.delete(bucket, object, &opts)
|
||||
.await
|
||||
.expect("absence does not report recovery")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-directory";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove upload metadata");
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write leftover part");
|
||||
}
|
||||
let metadata_directory = temp_dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&metadata_path);
|
||||
tokio::fs::create_dir_all(&metadata_directory)
|
||||
.await
|
||||
.expect("replace one missing metadata file with a directory");
|
||||
let (_, errors) =
|
||||
SetDisks::read_all_raw_file_info(&set.disks.read().await, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
assert!(
|
||||
errors.iter().all(|error| matches!(error, Some(DiskError::FileNotFound))),
|
||||
"raw metadata reads alone must reproduce the directory-as-absence case: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one physical metadata directory must block cleanup on every disk"
|
||||
);
|
||||
assert!(metadata_directory.is_dir(), "failed preflight must preserve the metadata directory");
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every leftover part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum OrphanMultipartLockDomainCase {
|
||||
Shared,
|
||||
Independent,
|
||||
PartiallyOverlapping,
|
||||
}
|
||||
|
||||
async fn check_orphan_multipart_distributed_lock_owners(case: OrphanMultipartLockDomainCase) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let first_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_clients = match case {
|
||||
OrphanMultipartLockDomainCase::Shared => vec![first_client.clone()],
|
||||
OrphanMultipartLockDomainCase::Independent => vec![second_client],
|
||||
OrphanMultipartLockDomainCase::PartiallyOverlapping => vec![first_client.clone(), second_client],
|
||||
};
|
||||
let (first_dirs, first_disks, first_set) =
|
||||
hermetic_set_disks_with_lockers_and_ctx(4, 0, 2, vec![first_client], ctx.clone()).await;
|
||||
let (second_dirs, second_disks, second_set) = hermetic_set_disks_with_lockers_and_ctx(4, 1, 2, second_clients, ctx).await;
|
||||
let bucket = "orphan-multipart-dist-locks";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&first_disks, bucket).await;
|
||||
make_bucket_on_all(&second_disks, bucket).await;
|
||||
let upload_id = runtime_sources::deployment_upload_id(&format!(
|
||||
"{}x{}",
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc().unix_timestamp_nanos()
|
||||
));
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
let staging = directory.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path);
|
||||
tokio::fs::create_dir_all(&staging)
|
||||
.await
|
||||
.expect("create exact orphan staging");
|
||||
tokio::fs::write(staging.join("part.late"), b"late part")
|
||||
.await
|
||||
.expect("write orphan part");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
let first_guard = first_set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("first domain has an orphan");
|
||||
let second_result = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
second_set.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, std::slice::from_ref(&first_guard)),
|
||||
)
|
||||
.await
|
||||
.expect("lock-domain admission must not wait on a guard it already owns");
|
||||
if case == OrphanMultipartLockDomainCase::PartiallyOverlapping {
|
||||
let error = match second_result {
|
||||
Ok(_) => panic!("one existing quorum cannot authorize a partially overlapping domain"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("partially overlapping lock domains"));
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
assert!(
|
||||
directory
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_path)
|
||||
.join("part.late")
|
||||
.exists(),
|
||||
"uncertain lock coverage must preserve every physical part"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let second_guard = second_result.expect("the second pool has a proven lock domain");
|
||||
assert_eq!(
|
||||
Arc::ptr_eq(&first_guard.lock_guard, &second_guard.lock_guard),
|
||||
case == OrphanMultipartLockDomainCase::Shared,
|
||||
"only the same exact backend may share a real upload guard across pool labels"
|
||||
);
|
||||
let second_namespace = second_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut second_writer = Box::pin(second_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut second_writer).is_pending());
|
||||
if case == OrphanMultipartLockDomainCase::Shared {
|
||||
drop(first_guard);
|
||||
assert!(
|
||||
futures::poll!(&mut second_writer).is_pending(),
|
||||
"the second pool must retain the shared real owner after the first guard drops"
|
||||
);
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("the shared backend should release after the last owner")
|
||||
.expect("shared-domain writer resumes"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let first_namespace = first_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut first_writer = Box::pin(first_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut first_writer).is_pending());
|
||||
drop(first_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), first_writer)
|
||||
.await
|
||||
.expect("first domain should release independently")
|
||||
.expect("first writer resumes"),
|
||||
);
|
||||
assert!(futures::poll!(&mut second_writer).is_pending(), "the second real owner must remain held");
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("second domain should release with its owner")
|
||||
.expect("second writer resumes"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_keeps_separate_distributed_lock_owners() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Independent).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_reuses_shared_distributed_owner_across_pools() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Shared).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_partially_overlapping_distributed_domains() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::PartiallyOverlapping).await;
|
||||
}
|
||||
|
||||
async fn stage_upload_with_create_opts(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
@@ -6370,6 +6905,76 @@ mod tests {
|
||||
.expect("abort should delete the upload after UploadPart releases the lock");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
|
||||
for pause in [
|
||||
MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-cancelled-part-commit-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, pause);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the request waiter should remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let mut abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut abort).await.is_err(),
|
||||
"Abort must not acquire the upload lock while a cancelled request's commit is paused at {pause:?}"
|
||||
);
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(10), abort)
|
||||
.await
|
||||
.expect("Abort should finish after the commit drains")
|
||||
.expect("Abort task should not panic")
|
||||
.expect("Abort should delete the committed part staging");
|
||||
for disk in &disk_stores {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the upload directory must be absent after the commit and Abort finish"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_different_part_numbers_commit_concurrently() {
|
||||
@@ -9799,6 +10404,18 @@ mod tests {
|
||||
"the post-commit crash must leave the upload listable for reclamation"
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("the committed upload has no staging metadata on any disk");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recover completed upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
// The commit rename already consumed the upload's xl.meta, so the
|
||||
// retry cannot read staging. It must still return the committed
|
||||
// object: AWS CompleteMultipartUpload is idempotent for the same
|
||||
@@ -9817,21 +10434,6 @@ mod tests {
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"the idempotent retry must reclaim staging left behind by the crash"
|
||||
);
|
||||
|
||||
// Reclaim the leftover exactly as the production tail does: delete_all
|
||||
// on the upload path (abort_multipart_upload cannot — the upload's
|
||||
// metadata is gone, so it too answers InvalidUploadID).
|
||||
let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
|
||||
set_disks
|
||||
.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
|
||||
.await
|
||||
.expect("sweeping the leftover upload staging should succeed");
|
||||
assert!(
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"reclaiming the upload must drop it from the listing"
|
||||
);
|
||||
let (body_after, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -9878,6 +10480,18 @@ mod tests {
|
||||
.expect("the failed post-commit completion should release its object guard"),
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("committed staging has no upload metadata");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("orphan recovery must only remove upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
let (body, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body, new, "the committed replacement must remain readable after the crash");
|
||||
let mut receipts = 0;
|
||||
|
||||
@@ -1152,7 +1152,7 @@ impl ECStore {
|
||||
let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
let result = match quorum {
|
||||
BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
|
||||
BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => set.get_bucket_info(bucket, opts).await,
|
||||
};
|
||||
(pool_index, set_index, result)
|
||||
}))
|
||||
@@ -1273,7 +1273,7 @@ impl ECStore {
|
||||
..Default::default()
|
||||
},
|
||||
SCANNER_BUCKET_LIST_SET_CONCURRENCY,
|
||||
BucketInfoQuorum::Write,
|
||||
BucketInfoQuorum::HealCandidates,
|
||||
selector,
|
||||
)
|
||||
.await
|
||||
@@ -2283,8 +2283,11 @@ mod tests {
|
||||
.list_buckets_for_admin_heal(None)
|
||||
.await
|
||||
.expect("online disk enumeration");
|
||||
assert!(!global.topology_complete, "a filtered sub-quorum bucket must invalidate coverage");
|
||||
assert_eq!(global.buckets.iter().map(|b| b.name.as_str()).collect::<Vec<_>>(), ["healthy-bucket"]);
|
||||
assert!(global.topology_complete, "every disk answered the candidate inventory");
|
||||
assert_eq!(
|
||||
global.buckets.iter().map(|b| b.name.as_str()).collect::<Vec<_>>(),
|
||||
["healthy-bucket", "partial-bucket"]
|
||||
);
|
||||
let healthy = store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0)))
|
||||
.await
|
||||
@@ -2296,11 +2299,8 @@ mod tests {
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(1, 0)))
|
||||
.await
|
||||
.expect("partial scope");
|
||||
assert!(!partial.topology_complete);
|
||||
assert!(
|
||||
partial.buckets.is_empty(),
|
||||
"missing buckets cannot be mistaken for a complete empty namespace"
|
||||
);
|
||||
assert!(partial.topology_complete);
|
||||
assert_eq!(partial.buckets[0].name, "partial-bucket");
|
||||
let invalid = store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 1)))
|
||||
.await
|
||||
|
||||
@@ -404,7 +404,17 @@ impl ECStore {
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealResultItem> {
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, mut opts| async move {
|
||||
if !opts.recreate && !opts.remove {
|
||||
let mut missing = false;
|
||||
for (pool, set) in store.heal_erasure_set_scopes(&opts).await? {
|
||||
missing |= store.pools[pool].get_disks(set).bucket_needs_reconstruction(&bucket).await?;
|
||||
}
|
||||
if missing {
|
||||
crate::bucket::metadata_sys::validate_bucket_reconstruction_in(&store.ctx, &bucket, expected).await?;
|
||||
opts.recreate = true;
|
||||
}
|
||||
}
|
||||
store.heal_bucket(&bucket, &opts).await
|
||||
})
|
||||
.await
|
||||
@@ -1451,6 +1461,178 @@ mod tests {
|
||||
.expect("minimal pool should build")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_recreates_missing_directories_at_current_incarnation() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "admin-bucket-recovery";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a committed bucket generation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("persisted generation");
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove only an empty fixture bucket directory");
|
||||
}
|
||||
store
|
||||
.heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.expect("default administrator recovery should use committed generation authority");
|
||||
for disk in 0..4 {
|
||||
assert!(root.path().join(format!("pool0-disk{disk}/{bucket}")).is_dir());
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("recovered generation"),
|
||||
incarnation
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_rejects_missing_corrupt_uncommitted_and_retired_authority() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
for state in ["missing", "corrupt", "uncommitted", "mismatched", "retired"] {
|
||||
let bucket = format!("recovery-authority-{state}");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create fixture generation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("fixture identity");
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("committed metadata");
|
||||
assert!(persisted && metadata.bucket_creation_committed);
|
||||
let path = metadata.save_file_path();
|
||||
match state {
|
||||
"missing" => delete_config(store.clone(), &path).await.expect("remove metadata authority"),
|
||||
"corrupt" => save_config(store.clone(), &path, b"corrupt bucket metadata".to_vec())
|
||||
.await
|
||||
.expect("corrupt authority"),
|
||||
"uncommitted" | "mismatched" => {
|
||||
if state == "uncommitted" {
|
||||
metadata.bucket_creation_committed = false;
|
||||
} else {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
metadata
|
||||
.save_with_store_committed(store.clone())
|
||||
.await
|
||||
.expect("inject inconsistent metadata authority");
|
||||
}
|
||||
"retired" => {
|
||||
// A returning disk may retain a directory after retirement was committed.
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), &bucket, incarnation, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist retirement evidence");
|
||||
}
|
||||
_ => unreachable!("fixed fixture states"),
|
||||
}
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("lose fixture directories");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.heal_bucket_at_incarnation(&bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"must reject {state}"
|
||||
);
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
|
||||
"{state} cannot authorize reconstruction"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_rejects_offline_disks_and_fully_missing_sets() {
|
||||
for offline in [false, true] {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "recovery-unproven-set";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
|
||||
let first_missing = usize::from(offline);
|
||||
for disk in first_missing..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove fixture directories");
|
||||
}
|
||||
if offline {
|
||||
let disk = store.pools[0].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("online fixture disk");
|
||||
disk.close().await.expect("stop disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for disk in first_missing..4 {
|
||||
assert!(!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists());
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_dry_run_preserves_missing_directories() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "recovery-dry-run";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove fixture directories");
|
||||
}
|
||||
store
|
||||
.heal_bucket_at_incarnation(
|
||||
bucket,
|
||||
incarnation,
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dry run can inspect eligible recovery");
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
|
||||
"dry run must not reconstruct directories"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
|
||||
|
||||
@@ -5030,6 +5030,339 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_recovers_exact_orphan_before_decommission_drains() {
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create orphan recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-drain", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "exact-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |pool_index: usize, disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(0, disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("logical upload listing")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert!(store.ensure_decommission_multipart_uploads_drained_for_test(0).await.is_err());
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, "different-key", &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the same UploadID for a different key must not recover this path"
|
||||
);
|
||||
|
||||
// A minority metadata copy in another pool blocks recovery everywhere.
|
||||
let conflicting_metadata = staging_dir(1, 0).join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(conflicting_metadata.parent().expect("metadata parent"))
|
||||
.await
|
||||
.expect("create conflicting pool staging");
|
||||
tokio::fs::write(&conflicting_metadata, b"unreadable metadata")
|
||||
.await
|
||||
.expect("write conflicting metadata");
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"no pool may be cleaned until every pool proves absence"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(staging_dir(0, disk_index).join("part.1").exists());
|
||||
}
|
||||
tokio::fs::remove_file(conflicting_metadata)
|
||||
.await
|
||||
.expect("remove conflicting fixture");
|
||||
let (paused, release) = crate::store::multipart::pause_orphan_multipart_cleanup_for_test(&bucket);
|
||||
let recovery_store = store.clone();
|
||||
let recovery_bucket = bucket.clone();
|
||||
let recovery_upload = upload.upload_id.clone();
|
||||
let mut recovery = tokio::spawn(async move {
|
||||
recovery_store
|
||||
.abort_multipart_upload(&recovery_bucket, object, &recovery_upload, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
paused = paused => paused.expect("cleanup owner announces arrival"),
|
||||
result = &mut recovery => panic!("recovery returned before physical cleanup: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("recovery should reach physical cleanup");
|
||||
recovery.abort();
|
||||
assert!(recovery.await.expect_err("request waiter is cancellable").is_cancelled());
|
||||
let upload_lock = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_MULTIPART_BUCKET, &format!("{upload_sha}/{upload_uuid}"))
|
||||
.await
|
||||
.expect("same upload namespace");
|
||||
let mut upload_writer = Box::pin(upload_lock.get_write_lock(Duration::from_secs(10)));
|
||||
let mut bucket_writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
futures::poll!(&mut upload_writer).is_pending(),
|
||||
"cancelled recovery must retain the upload lock"
|
||||
);
|
||||
assert!(
|
||||
futures::poll!(&mut bucket_writer).is_pending(),
|
||||
"cancelled recovery must retain the real bucket guard"
|
||||
);
|
||||
release.send(()).expect("allow physical cleanup to finish");
|
||||
let (upload_guard, bucket_guard) =
|
||||
tokio::time::timeout(Duration::from_secs(10), async { tokio::join!(upload_writer, bucket_writer) })
|
||||
.await
|
||||
.expect("both guards should release after physical cleanup");
|
||||
drop(upload_guard.expect("upload writer resumes"));
|
||||
drop(bucket_guard.expect("bucket writer resumes"));
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(pool_index, disk_index).exists(), "every checked staging path must be absent");
|
||||
}
|
||||
}
|
||||
store
|
||||
.ensure_decommission_multipart_uploads_drained_for_test(0)
|
||||
.await
|
||||
.expect("physical decommission gate should open after recovery");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"an absent upload must not claim to have recovered anything"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_keeps_no_such_upload_while_a_disk_is_offline() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create offline-disk recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-offline", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "offline-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
|
||||
// One offline disk means absence cannot be proven. The idempotent Abort
|
||||
// must keep answering NoSuchUpload instead of surfacing a disk error,
|
||||
// and must not delete anything.
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let offline = set.disks.write().await[3].take();
|
||||
assert!(offline.is_some(), "fixture disk must have been online");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"unprovable absence must keep NoSuchUpload while a disk is offline"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(
|
||||
staging_dir(disk_index).join("part.1").exists(),
|
||||
"no staging may be deleted while a disk is offline"
|
||||
);
|
||||
}
|
||||
|
||||
set.disks.write().await[3] = offline;
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery resumes once every disk is observable");
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(disk_index).exists(), "recovered staging must be absent");
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn orphan_multipart_recovery_preserves_current_bucket_incarnation() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create bucket lifetime recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-bucket-lifetime", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-lifetime-{}", uuid::Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let original_incarnation = store.bucket_incarnation_id(&bucket).await.expect("original incarnation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create original upload");
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort original upload");
|
||||
store
|
||||
.delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(store.bucket_incarnation_id(&bucket).await.expect("new incarnation"), original_incarnation);
|
||||
let mut reader = PutObjReader::from_vec(b"current bucket object".to_vec());
|
||||
store
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write current bucket object");
|
||||
let current_upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create current incarnation upload for the same key");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let leftover = temp_dir
|
||||
.path()
|
||||
.join("pool0/set0/disk0")
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(upload_sha)
|
||||
.join(upload_uuid)
|
||||
.join("part.1");
|
||||
tokio::fs::create_dir_all(leftover.parent().expect("staging parent"))
|
||||
.await
|
||||
.expect("create old incarnation staging");
|
||||
tokio::fs::write(&leftover, b"previous bucket lifetime")
|
||||
.await
|
||||
.expect("write previous incarnation fixture");
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(original_incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(uuid::Uuid::nil()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale or nil current-bucket identity must fail closed"
|
||||
);
|
||||
assert!(leftover.exists(), "failed bucket lifetime proof must preserve the path");
|
||||
}
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("exact metadata-less staging from an old incarnation is unreferenced");
|
||||
assert!(!leftover.exists());
|
||||
store
|
||||
.get_multipart_info(&bucket, object, ¤t_upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery must preserve the current incarnation's active upload");
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("current object must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read current object");
|
||||
assert_eq!(body, b"current bucket object");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn active_multipart_upload_routes_before_faulted_suspended_source() {
|
||||
@@ -18591,6 +18924,70 @@ mod tests {
|
||||
assert_eq!(body, b"new-generation");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn cancelled_ecstore_put_part_keeps_bucket_lifecycle_lock_through_commit() {
|
||||
for pause in [
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create cancelled multipart lifecycle store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "cancelled-multipart-lifecycle", &[4, 4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("cancelled-part-lifecycle-{}", Uuid::new_v4());
|
||||
let object = "part.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for multipart lifecycle cancellation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload through the owning ECStore");
|
||||
let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put_bucket = bucket.clone();
|
||||
let put_upload_id = upload.upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(&put_bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the ECStore request waiter must remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err(),
|
||||
"canceling UploadPart at {pause:?} must not release its bucket lifecycle owner before the commit drains"
|
||||
);
|
||||
barrier.release();
|
||||
let lifecycle_guard = tokio::time::timeout(Duration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("the bucket lifecycle writer must resume after the part commit drains")
|
||||
.expect("acquire the bucket lifecycle write lock after cancellation");
|
||||
drop(lifecycle_guard);
|
||||
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the cancelled request's owned part commit must finish successfully");
|
||||
assert_eq!(parts.parts.len(), 1, "the commit must publish exactly the uploaded part at {pause:?}");
|
||||
assert_eq!(parts.parts[0].part_num, 1);
|
||||
assert_eq!(parts.parts[0].size, 4096);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
|
||||
|
||||
@@ -32,6 +32,42 @@ fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex<std::
|
||||
DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type OrphanMultipartCleanupPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
static ORPHAN_MULTIPART_CLEANUP_PAUSES: std::sync::OnceLock<
|
||||
std::sync::Mutex<std::collections::HashMap<String, OrphanMultipartCleanupPause>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pause_orphan_multipart_cleanup_for_test(
|
||||
bucket: &str,
|
||||
) -> (tokio::sync::oneshot::Receiver<()>, tokio::sync::oneshot::Sender<()>) {
|
||||
let (arrived_tx, arrived_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let previous = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.insert(bucket.to_owned(), (arrived_tx, release_rx));
|
||||
assert!(previous.is_none(), "each bucket has at most one orphan cleanup pause");
|
||||
(arrived_rx, release_tx)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_orphan_multipart_cleanup(bucket: &str) {
|
||||
let pause = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.remove(bucket);
|
||||
if let Some((arrived, release)) = pause {
|
||||
let _ = arrived.send(());
|
||||
let _ = release.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result<bool> {
|
||||
let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked {
|
||||
message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(),
|
||||
@@ -654,7 +690,10 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -750,19 +789,12 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
// TODO(backlog): defer DeleteUploadID to background for faster abort response
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
|
||||
}
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
let pool = &self.pools[pool_idx];
|
||||
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, &opts).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) => {
|
||||
//
|
||||
@@ -775,7 +807,92 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
self.recover_orphan_multipart_upload(bucket, object, upload_id, opts, bucket_lifecycle_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_orphan_multipart_upload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: ObjectOptions,
|
||||
bucket_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
|
||||
// A signal borrowed from an enclosing operation cannot keep its lock
|
||||
// alive when the request waiter disappears during physical cleanup.
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
let Some(expected_incarnation) = opts.expected_bucket_incarnation_id.filter(|id| !id.is_nil()) else {
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
// Recovery is a side path of an idempotent Abort. Until the first
|
||||
// deletion is issued, anything that stops staging absence from being
|
||||
// proven (an offline disk, lock contention, partially overlapping lock
|
||||
// domains, unreadable bucket metadata) keeps the upload reported as
|
||||
// gone, exactly as before recovery existed; only failures inside the
|
||||
// deletion task are request failures, because a retry can finish them.
|
||||
let (metadata, persisted) =
|
||||
match crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await {
|
||||
Ok(loaded) => loaded,
|
||||
Err(error) => {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: bucket metadata unavailable");
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
if !persisted
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.bucket_incarnation_sidecar
|
||||
|| metadata.bucket_incarnation_id != expected_incarnation
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle, then upload locks in pool-index order.
|
||||
// Prove metadata absence across every pool before deleting any staging.
|
||||
let mut cleanup_guards = Vec::with_capacity(self.pools.len());
|
||||
for pool in &self.pools {
|
||||
let guard = match pool
|
||||
.get_disks_by_key(object)
|
||||
.lock_orphan_multipart_cleanup(bucket, object, upload_id, &opts, &cleanup_guards)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
if !is_err_invalid_upload_id(&error) {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: staging absence not proven");
|
||||
}
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
cleanup_guards.push(guard);
|
||||
}
|
||||
let bucket = bucket.to_owned();
|
||||
let object = object.to_owned();
|
||||
let upload_id = upload_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
// Own both the real bucket guard and all upload guards until every
|
||||
// disk deletion settles, including after cancellation of the waiter.
|
||||
let _bucket_lifecycle_guard = bucket_lifecycle_guard;
|
||||
#[cfg(test)]
|
||||
pause_orphan_multipart_cleanup(&bucket).await;
|
||||
let mut recovered = false;
|
||||
for guard in &cleanup_guards {
|
||||
recovered |= guard.delete(&bucket, &object, &opts).await?;
|
||||
}
|
||||
if recovered {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StorageError::InvalidUploadID(bucket, object, upload_id))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::other_with_context("orphan multipart cleanup task failed", error))?
|
||||
}
|
||||
|
||||
pub(crate) async fn abort_multipart_upload_for_data_movement(
|
||||
|
||||
@@ -1697,7 +1697,9 @@ mod tests {
|
||||
#[serial_test::serial]
|
||||
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||
// Allow real fixture I/O to finish under load. Expire the operation
|
||||
// with virtual time only after its physical publication is paused.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
@@ -1731,7 +1733,7 @@ mod tests {
|
||||
.await
|
||||
.expect("bounded entry");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||
tokio::time::advance(std::time::Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||
.await
|
||||
|
||||
@@ -1400,6 +1400,15 @@ impl ErasureSetHealer {
|
||||
None => self.storage.heal_bucket(bucket, &bucket_opts).await?,
|
||||
};
|
||||
task.record_result_item(result).await;
|
||||
// Keep recovery in the checkpoint's admitted incarnation and repair
|
||||
// the publication bucket before adopting a survivor's delete proof.
|
||||
if let Some(result) = self
|
||||
.storage
|
||||
.recover_delete_transactions(bucket, None, incarnation, &self.heal_opts)
|
||||
.await?
|
||||
{
|
||||
task.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
// 1. get bucket info
|
||||
|
||||
@@ -13,6 +13,18 @@
|
||||
// limitations under the License.
|
||||
/// The automatic disk scanner: replacement discovery and unformatted-disk enqueue.
|
||||
use super::*;
|
||||
use crate::heal::DiskStore;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) fn extend_replacement_recovery_disks(
|
||||
inventory: &mut BTreeMap<String, DiskStore>,
|
||||
disks: impl IntoIterator<Item = DiskStore>,
|
||||
) {
|
||||
for disk in disks {
|
||||
// Preserve the local snapshot when later inventory repeats its endpoint.
|
||||
inventory.entry(disk.endpoint().to_string()).or_insert(disk);
|
||||
}
|
||||
}
|
||||
|
||||
impl HealManager {
|
||||
/// Start background task to auto scan local disks and enqueue erasure set heal requests
|
||||
@@ -87,7 +99,20 @@ impl HealManager {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
let mut local_endpoints = HashMap::with_capacity(local_disks.len());
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
local_endpoints.entry(endpoint.to_string()).or_insert(endpoint);
|
||||
}
|
||||
// Formatting removes a target from blank-disk discovery, but its
|
||||
// unfinished intent may still live on a remote survivor.
|
||||
let mut recovery_sets = local_disks
|
||||
.iter()
|
||||
.filter_map(|disk| {
|
||||
let endpoint = disk.endpoint();
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
let blocked_sets = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
@@ -97,6 +122,7 @@ impl HealManager {
|
||||
let mut retry_succeeded = HashSet::new();
|
||||
let mut retry_failed = HashSet::new();
|
||||
for set_disk_id in blocked_sets {
|
||||
recovery_sets.remove(&set_disk_id);
|
||||
let disks = match storage.replacement_intent_disks(&set_disk_id).await {
|
||||
Ok(disks) => disks,
|
||||
Err(error) => {
|
||||
@@ -235,11 +261,12 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
let mut recovery_disks = local_disks.clone();
|
||||
recovery_disks.extend(retry_recovery_disks);
|
||||
for set_disk_id in endpoints.keys() {
|
||||
let mut recovery_disks = BTreeMap::new();
|
||||
extend_replacement_recovery_disks(&mut recovery_disks, local_disks.iter().cloned());
|
||||
extend_replacement_recovery_disks(&mut recovery_disks, retry_recovery_disks);
|
||||
for set_disk_id in &recovery_sets {
|
||||
match storage.replacement_intent_disks(set_disk_id).await {
|
||||
Ok(disks) => recovery_disks.extend(disks),
|
||||
Ok(disks) => extend_replacement_recovery_disks(&mut recovery_disks, disks),
|
||||
Err(error) => {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
warn!(
|
||||
@@ -254,10 +281,6 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
recovery_disks.sort_by_key(|disk| disk.endpoint().to_string());
|
||||
recovery_disks
|
||||
.dedup_by(|left, right| left.endpoint().to_string() == right.endpoint().to_string());
|
||||
|
||||
// Once formatting succeeds a replacement is no longer
|
||||
// discoverable as UnformattedDisk. Re-admit exactly one
|
||||
// incomplete durable generation per set within its
|
||||
@@ -265,7 +288,7 @@ impl HealManager {
|
||||
// verified terminal cleanup. Multiple generations are a
|
||||
// durable conflict: leave every marker/state intact and
|
||||
// require reconciliation rather than choosing one.
|
||||
for disk in &recovery_disks {
|
||||
for (anchor, disk) in &recovery_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let disk_set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
@@ -351,7 +374,7 @@ impl HealManager {
|
||||
// A remote survivor may own the durable intent, but only the
|
||||
// node mounting every replacement target may advance or replay it.
|
||||
if state.replacement_targets.iter().any(|target| {
|
||||
!local_endpoints.iter().any(|endpoint| endpoint.to_string() == *target)
|
||||
!local_endpoints.contains_key(target)
|
||||
}) {
|
||||
deferred_recovery_sets.insert(state.set_disk_id.clone());
|
||||
continue;
|
||||
@@ -361,19 +384,20 @@ impl HealManager {
|
||||
}
|
||||
let mut selected = durable_recovery_candidates.remove(&task_id);
|
||||
let previous_set = selected.as_ref().map(|candidate| candidate.state.set_disk_id.clone());
|
||||
let candidate_set = state.set_disk_id.clone();
|
||||
let candidate = match ReplacementRecoveryCandidate::new(
|
||||
state.clone(),
|
||||
disk.endpoint().to_string(),
|
||||
disk.endpoint().is_local,
|
||||
state,
|
||||
anchor.clone(),
|
||||
endpoint.is_local,
|
||||
) {
|
||||
Ok(candidate) => candidate,
|
||||
Err(error) => {
|
||||
conflicted_recovery_tasks.insert(task_id.clone());
|
||||
conflicted_recovery_sets.insert(state.set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(candidate_set.clone());
|
||||
if let Some(previous_set) = &previous_set {
|
||||
conflicted_recovery_sets.insert(previous_set.clone());
|
||||
}
|
||||
block_recovery_set(&state.set_disk_id);
|
||||
block_recovery_set(&candidate_set);
|
||||
if let Some(previous_set) = &previous_set {
|
||||
block_recovery_set(previous_set);
|
||||
}
|
||||
@@ -432,12 +456,7 @@ impl HealManager {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(anchor_disk) = local_disks
|
||||
.iter()
|
||||
.chain(recovery_disks.iter())
|
||||
.find(|disk| disk.endpoint().to_string() == candidate.anchor)
|
||||
.cloned()
|
||||
else {
|
||||
let Some(anchor_disk) = recovery_disks.get(&candidate.anchor).cloned() else {
|
||||
deferred_recovery_sets.insert(candidate.state.set_disk_id.clone());
|
||||
continue;
|
||||
};
|
||||
@@ -530,12 +549,7 @@ impl HealManager {
|
||||
let targets = state
|
||||
.replacement_targets
|
||||
.iter()
|
||||
.filter_map(|target| {
|
||||
local_endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.to_string() == *target)
|
||||
.cloned()
|
||||
})
|
||||
.filter_map(|target| local_endpoints.get(target).cloned())
|
||||
.collect::<Vec<_>>();
|
||||
if targets.len() != state.replacement_targets.len() {
|
||||
deferred_recovery_sets.insert(state.set_disk_id.clone());
|
||||
@@ -607,20 +621,9 @@ impl HealManager {
|
||||
}
|
||||
|
||||
for (set_disk_id, (_, targets, _, _)) in &durable_recoveries {
|
||||
let expected = targets.iter().map(ToString::to_string).collect::<HashSet<_>>();
|
||||
let observed = endpoints
|
||||
.get(set_disk_id)
|
||||
.map(|endpoints| endpoints.iter().map(ToString::to_string).collect::<HashSet<_>>())
|
||||
.unwrap_or_default();
|
||||
if !observed.is_subset(&expected) {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
continue;
|
||||
}
|
||||
endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone());
|
||||
// The durable owner must finish its exact target scope before
|
||||
// newly discovered replacements can start another generation.
|
||||
endpoints.insert(set_disk_id.clone(), targets.clone());
|
||||
}
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! or deletion.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::outcome::HealExecutionOutcome;
|
||||
use crate::heal::resume::CheckpointManager;
|
||||
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
|
||||
use crate::heal::{DiskStore, RUSTFS_META_BUCKET};
|
||||
@@ -36,12 +37,15 @@ const ROOT_RECOVERY_PREFIX: &str = "root-heal-";
|
||||
const ROOT_TERMINAL_PREFIX: &str = "terminal-root-heal-";
|
||||
const LEGACY_ROOT_RECOVERY_SCHEMA: u32 = 1;
|
||||
const SCOPED_ROOT_RECOVERY_SCHEMA: u32 = 2;
|
||||
const ROOT_RECOVERY_SCHEMA: u32 = 3;
|
||||
const BUCKET_ROOT_RECOVERY_SCHEMA: u32 = 3;
|
||||
const ROOT_RECOVERY_SCHEMA: u32 = 4;
|
||||
const ROOT_TERMINAL_SCHEMA: u32 = 1;
|
||||
const ROOT_QUARANTINE_INTENT_PREFIX: &str = "quarantined-root-heal-intent-";
|
||||
const ROOT_QUARANTINE_TERMINAL_PREFIX: &str = "quarantined-root-heal-terminal-";
|
||||
const ROOT_QUARANTINE_SCHEMA: u32 = 1;
|
||||
pub(super) const MAX_ROOT_RECOVERY_BYTES: usize = 64 * 1024;
|
||||
// Cluster checkpoints additionally retain a frozen inventory and the bounded outcome window.
|
||||
pub(super) const MAX_CLUSTER_RECOVERY_BYTES: usize = 16 * 1024 * 1024;
|
||||
const ROOT_TERMINAL_GC_SCAN_BUDGET: usize = 1024;
|
||||
const ROOT_TERMINAL_GC_DELETE_BUDGET: usize = 64;
|
||||
|
||||
@@ -237,7 +241,83 @@ fn default_recovery_heal_type() -> RecoveryHealType {
|
||||
RecoveryHealType::Cluster
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct AdminClusterScope {
|
||||
#[serde(deserialize_with = "decode_cluster_buckets")]
|
||||
pub(crate) buckets: std::collections::BTreeMap<String, Uuid>,
|
||||
pub(crate) next_bucket_index: usize,
|
||||
pub(crate) outcome: HealTaskOutcome,
|
||||
pub(crate) progress: HealProgress,
|
||||
}
|
||||
|
||||
fn decode_cluster_buckets<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<std::collections::BTreeMap<String, Uuid>, D::Error> {
|
||||
struct Buckets;
|
||||
impl<'de> serde::de::Visitor<'de> for Buckets {
|
||||
type Value = std::collections::BTreeMap<String, Uuid>;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("unique bucket generation bindings")
|
||||
}
|
||||
fn visit_map<M: serde::de::MapAccess<'de>>(self, mut map: M) -> std::result::Result<Self::Value, M::Error> {
|
||||
let mut buckets = std::collections::BTreeMap::new();
|
||||
while let Some((bucket, incarnation)) = map.next_entry::<String, Uuid>()? {
|
||||
if buckets.insert(bucket, incarnation).is_some() {
|
||||
return Err(serde::de::Error::custom("duplicate administrator cluster bucket"));
|
||||
}
|
||||
}
|
||||
Ok(buckets)
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_map(Buckets)
|
||||
}
|
||||
|
||||
impl AdminClusterScope {
|
||||
pub(crate) fn checkpoint_interval(&self) -> usize {
|
||||
// Cap full-inventory rewrites per traversal, including many empty buckets.
|
||||
self.buckets.len().div_ceil(32).max(1)
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<()> {
|
||||
if self.next_bucket_index > self.buckets.len()
|
||||
|| self.outcome.execution != HealExecutionOutcome::Running
|
||||
|| self.outcome.coverage != crate::heal::outcome::HealTraversalCoverage::Partial
|
||||
|| self.outcome.counters.failed != 0
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Invalid administrator cluster checkpoint progress".to_string()));
|
||||
}
|
||||
for (bucket, incarnation) in &self.buckets {
|
||||
if bucket.contains('/')
|
||||
|| bucket.trim() != bucket
|
||||
|| crate::heal::storage_api::owner::ecstore_is_reserved_or_invalid_bucket(bucket, false)
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Invalid administrator cluster bucket name".to_string()));
|
||||
}
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster checkpoint lacks a bucket incarnation".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let completed_through = self
|
||||
.next_bucket_index
|
||||
.checked_sub(1)
|
||||
.and_then(|index| self.buckets.keys().nth(index));
|
||||
if self.outcome.objects.iter().any(|item| {
|
||||
completed_through.is_none_or(|last| item.identity.bucket > *last)
|
||||
|| self.buckets.get(&item.identity.bucket).copied() != item.identity.bucket_incarnation_id
|
||||
}) || (self.next_bucket_index == 0 && self.outcome.counters.processed != 0)
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster checkpoint outcome is outside its completed scope".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RootHealIntent {
|
||||
schema: u32,
|
||||
@@ -246,6 +326,8 @@ struct RootHealIntent {
|
||||
heal_type: RecoveryHealType,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
bucket_incarnation_id: Option<Uuid>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
cluster_scope: Option<AdminClusterScope>,
|
||||
#[serde(deserialize_with = "decode_options")]
|
||||
options: HealOptions,
|
||||
priority: HealPriority,
|
||||
@@ -292,6 +374,26 @@ struct RootHealSource {
|
||||
source_oversized: bool,
|
||||
}
|
||||
|
||||
impl RootHealQuarantine {
|
||||
fn matches_source(&self, source: &RootHealSource) -> bool {
|
||||
if self.source_prefix_len == source.bytes.len()
|
||||
&& self.source_prefix_sha256 == source.source_prefix_sha256
|
||||
&& self.source_oversized == source.source_oversized
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// Before schema 4, oversized intents retained only the 64 KiB prefix.
|
||||
// Keep that quarantine fence across upgrades without reinterpreting its source.
|
||||
self.record_kind == RootHealRecordKind::Intent
|
||||
&& self.source_oversized
|
||||
&& self.source_prefix_len == MAX_ROOT_RECOVERY_BYTES + 1
|
||||
&& source
|
||||
.bytes
|
||||
.get(..self.source_prefix_len)
|
||||
.is_some_and(|prefix| Sha256::digest(prefix).as_slice() == self.source_prefix_sha256.as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RootHealInventory {
|
||||
pub(super) requests: Vec<HealRequest>,
|
||||
@@ -362,12 +464,38 @@ impl RootHealTerminal {
|
||||
}
|
||||
|
||||
impl RootHealIntent {
|
||||
fn encode(&self) -> Result<EcstoreDiskBytes> {
|
||||
let limit = match &self.cluster_scope {
|
||||
// Reserve headroom for the bounded outcome window and progress so
|
||||
// completing work cannot overflow an inventory admitted at the limit.
|
||||
Some(scope) if scope.next_bucket_index == 0 => MAX_CLUSTER_RECOVERY_BYTES - 1024 * 1024,
|
||||
Some(_) => MAX_CLUSTER_RECOVERY_BYTES,
|
||||
None => MAX_ROOT_RECOVERY_BYTES,
|
||||
};
|
||||
encode_root_record(self, limit)
|
||||
}
|
||||
|
||||
fn validate_task(&self, task: &HealTask) -> Result<()> {
|
||||
let mut options = self.options.clone();
|
||||
options.timeout = task.options.timeout;
|
||||
if HealType::from(self.heal_type.clone()) != task.heal_type
|
||||
|| self.bucket_incarnation_id != task.bucket_incarnation_id
|
||||
|| self.created_at != task.created_at
|
||||
|| self.priority != task.priority
|
||||
|| options != task.options
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn from_request(request: &HealRequest) -> Result<Self> {
|
||||
Ok(Self {
|
||||
schema: ROOT_RECOVERY_SCHEMA,
|
||||
task_id: request.id.clone(),
|
||||
heal_type: RecoveryHealType::try_from(&request.heal_type)?,
|
||||
bucket_incarnation_id: request.bucket_incarnation_id,
|
||||
cluster_scope: None,
|
||||
options: request.options.clone(),
|
||||
priority: request.priority,
|
||||
retry_attempts: request.retry_attempts,
|
||||
@@ -386,6 +514,32 @@ impl RootHealIntent {
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_root_record(value: &impl Serialize, limit: usize) -> Result<EcstoreDiskBytes> {
|
||||
struct BoundedWriter {
|
||||
bytes: Vec<u8>,
|
||||
limit: usize,
|
||||
}
|
||||
impl std::io::Write for BoundedWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("root heal record exceeds its size limit"));
|
||||
}
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut writer = BoundedWriter {
|
||||
bytes: Vec::new(),
|
||||
limit,
|
||||
};
|
||||
serde_json::to_writer(&mut writer, value)
|
||||
.map_err(|error| Error::Serialization(format!("Serialize root heal record: {error}")))?;
|
||||
Ok(writer.bytes.into())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RootHealRecovery {
|
||||
mutation: Mutex<()>,
|
||||
@@ -490,19 +644,27 @@ async fn read_bounded(disk: &DiskStore, path: &str) -> Result<Option<EcstoreDisk
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
reader
|
||||
.take(u64::try_from(MAX_ROOT_RECOVERY_BYTES + 1).map_err(Error::other)?)
|
||||
.take(u64::try_from(root_record_read_limit(path) + 1).map_err(Error::other)?)
|
||||
.read_to_end(&mut bytes)
|
||||
.await?;
|
||||
Ok(Some(bytes.into()))
|
||||
}
|
||||
|
||||
fn root_record_read_limit(path: &str) -> usize {
|
||||
if path.starts_with(ROOT_RECOVERY_PREFIX) {
|
||||
MAX_CLUSTER_RECOVERY_BYTES
|
||||
} else {
|
||||
MAX_ROOT_RECOVERY_BYTES
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_source(disk: &DiskStore, path: &str) -> Result<Option<RootHealSource>> {
|
||||
let Some(bytes) = read_bounded(disk, path).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(RootHealSource {
|
||||
source_prefix_sha256: Sha256::digest(&bytes).to_vec(),
|
||||
source_oversized: bytes.len() > MAX_ROOT_RECOVERY_BYTES,
|
||||
source_oversized: bytes.len() > root_record_read_limit(path),
|
||||
bytes,
|
||||
}))
|
||||
}
|
||||
@@ -544,7 +706,7 @@ fn encode_quarantine(
|
||||
|
||||
fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
|
||||
let _ = intent_path(task_id)?;
|
||||
if bytes.len() > MAX_ROOT_RECOVERY_BYTES {
|
||||
if bytes.len() > MAX_CLUSTER_RECOVERY_BYTES {
|
||||
return Err(Error::Other(format!("Root heal recovery record exceeds its size limit {task_id}")));
|
||||
}
|
||||
let mut intent: RootHealIntent = serde_json::from_slice(bytes)
|
||||
@@ -556,9 +718,25 @@ fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
|
||||
LEGACY_ROOT_RECOVERY_SCHEMA
|
||||
if intent.heal_type == RecoveryHealType::Cluster && intent.bucket_incarnation_id.is_none() => {}
|
||||
SCOPED_ROOT_RECOVERY_SCHEMA if intent.bucket_incarnation_id.is_none() => {}
|
||||
BUCKET_ROOT_RECOVERY_SCHEMA if intent.cluster_scope.is_none() => {}
|
||||
ROOT_RECOVERY_SCHEMA => {}
|
||||
_ => return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))),
|
||||
}
|
||||
if intent.heal_type == RecoveryHealType::Cluster && intent.schema != ROOT_RECOVERY_SCHEMA {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Legacy cluster heal has no frozen bucket generations; start a fresh task".to_string(),
|
||||
));
|
||||
}
|
||||
match &intent.cluster_scope {
|
||||
Some(scope) if intent.schema == ROOT_RECOVERY_SCHEMA && intent.heal_type == RecoveryHealType::Cluster => {
|
||||
scope.validate()?;
|
||||
}
|
||||
Some(_) => return Err(Error::InvalidCheckpoint("Unexpected administrator cluster scope".to_string())),
|
||||
None if bytes.len() > MAX_ROOT_RECOVERY_BYTES => {
|
||||
return Err(Error::InvalidCheckpoint("Root heal recovery record exceeds its size limit".to_string()));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if !matches!(intent.heal_type, RecoveryHealType::Bucket { .. }) && intent.bucket_incarnation_id.is_some() {
|
||||
return Err(Error::Other(format!(
|
||||
"Unexpected bucket incarnation in root heal recovery record {task_id}"
|
||||
@@ -723,10 +901,7 @@ impl RootHealRecovery {
|
||||
}
|
||||
};
|
||||
let marker = decode_quarantine(task_id, record_kind, &marker_bytes)?;
|
||||
if marker.source_prefix_len != source.bytes.len()
|
||||
|| marker.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.source_oversized != source.source_oversized
|
||||
{
|
||||
if !marker.matches_source(source) {
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
Ok(marker)
|
||||
@@ -808,9 +983,7 @@ impl RootHealRecovery {
|
||||
if let Some((owner, source)) = terminal_raw.into_iter().next() {
|
||||
if let Some(marker) = markers.get(&(RootHealRecordKind::Terminal, task_id.clone())) {
|
||||
if EcstoreDiskAPI::endpoint(marker.0.as_ref()) != EcstoreDiskAPI::endpoint(owner.as_ref())
|
||||
|| marker.1.source_prefix_len != source.bytes.len()
|
||||
|| marker.1.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.1.source_oversized != source.source_oversized
|
||||
|| !marker.1.matches_source(&source)
|
||||
{
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
@@ -849,9 +1022,7 @@ impl RootHealRecovery {
|
||||
};
|
||||
if let Some(marker) = markers.get(&(RootHealRecordKind::Intent, task_id.clone())) {
|
||||
if EcstoreDiskAPI::endpoint(marker.0.as_ref()) != EcstoreDiskAPI::endpoint(owner.as_ref())
|
||||
|| marker.1.source_prefix_len != source.bytes.len()
|
||||
|| marker.1.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.1.source_oversized != source.source_oversized
|
||||
|| !marker.1.matches_source(&source)
|
||||
{
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
@@ -924,18 +1095,29 @@ impl RootHealRecovery {
|
||||
if request.options.no_lock {
|
||||
return Err(Error::Other("Administrator root heal cannot skip namespace locking".to_string()));
|
||||
}
|
||||
let intent = RootHealIntent::from_request(request)?;
|
||||
let mut intent = RootHealIntent::from_request(request)?;
|
||||
intent.heal_type.validate()?;
|
||||
let bytes =
|
||||
serde_json::to_vec(&intent).map_err(|error| Error::Other(format!("Serialize root heal recovery record: {error}")))?;
|
||||
let path = intent_path(&request.id)?;
|
||||
if let Some((disk, expected)) = existing {
|
||||
let previous = decode_intent(&request.id, &expected)?;
|
||||
let mut previous_options = previous.options.clone();
|
||||
previous_options.timeout = intent.options.timeout;
|
||||
if previous.heal_type != intent.heal_type
|
||||
|| previous.bucket_incarnation_id != intent.bucket_incarnation_id
|
||||
|| previous.created_at != intent.created_at
|
||||
|| previous.priority != intent.priority
|
||||
|| previous_options != intent.options
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Root heal recovery owner changed during handoff".to_string()));
|
||||
}
|
||||
intent.cluster_scope = previous.cluster_scope;
|
||||
let bytes = intent.encode()?;
|
||||
return match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
Some(expected),
|
||||
Some(bytes.into()),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -944,19 +1126,15 @@ impl RootHealRecovery {
|
||||
};
|
||||
}
|
||||
|
||||
let bytes = intent.encode()?;
|
||||
|
||||
if disks.is_empty() {
|
||||
return Err(Error::Other("No local disk available for root heal shutdown recovery".to_string()));
|
||||
}
|
||||
let mut last_not_committed = None;
|
||||
for disk in &disks {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
Some(bytes.clone().into()),
|
||||
)
|
||||
.await
|
||||
match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, &path, None, Some(bytes.clone()))
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()),
|
||||
Ok(_) => return Err(Error::Other(format!("Root heal recovery record changed for {}", request.id))),
|
||||
@@ -970,6 +1148,63 @@ impl RootHealRecovery {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cluster_scope(&self, task: &HealTask) -> Result<Option<AdminClusterScope>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(None);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let (_, bytes) = Self::find(&self.disks().await?, &task.id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Administrator cluster intent is missing".to_string()))?;
|
||||
let intent = decode_intent(&task.id, &bytes)?;
|
||||
intent.validate_task(task)?;
|
||||
Ok(intent.cluster_scope)
|
||||
}
|
||||
|
||||
pub(crate) async fn checkpoint_cluster_scope(&self, task: &HealTask, scope: &AdminClusterScope) -> Result<()> {
|
||||
scope.validate()?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(());
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let (disk, expected) = Self::find(&self.disks().await?, &task.id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Administrator cluster intent is missing".to_string()))?;
|
||||
let mut intent = decode_intent(&task.id, &expected)?;
|
||||
intent.validate_task(task)?;
|
||||
if intent.heal_type != RecoveryHealType::Cluster {
|
||||
return Err(Error::InvalidCheckpoint("Cluster scope requires a cluster task".to_string()));
|
||||
}
|
||||
if let Some(previous) = &intent.cluster_scope {
|
||||
if previous.buckets != scope.buckets
|
||||
|| scope.next_bucket_index < previous.next_bucket_index
|
||||
|| scope.next_bucket_index > previous.next_bucket_index.saturating_add(scope.checkpoint_interval())
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster scope changed during recovery".to_string(),
|
||||
));
|
||||
}
|
||||
} else if scope.next_bucket_index != 0 {
|
||||
return Err(Error::InvalidCheckpoint("Administrator cluster scope was not initialized".to_string()));
|
||||
}
|
||||
intent.cluster_scope = Some(scope.clone());
|
||||
let bytes = intent.encode()?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task.id)?,
|
||||
Some(expected),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(()),
|
||||
_ => Err(Error::InvalidCheckpoint("Administrator cluster checkpoint owner changed".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn remove(&self, task_id: &str, heal_type: &HealType, source: HealRequestSource) -> Result<bool> {
|
||||
if !is_admin_heal_recovery(heal_type, source) {
|
||||
return Ok(false);
|
||||
@@ -1015,16 +1250,7 @@ impl RootHealRecovery {
|
||||
return Ok(());
|
||||
};
|
||||
let mut intent = decode_intent(&task.id, &expected)?;
|
||||
if HealType::from(intent.heal_type.clone()) != task.heal_type
|
||||
|| intent.bucket_incarnation_id != task.bucket_incarnation_id
|
||||
{
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
let mut expected_options = intent.options.clone();
|
||||
expected_options.timeout = task.options.timeout;
|
||||
if intent.created_at != task.created_at || intent.priority != task.priority || expected_options != task.options {
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
intent.validate_task(task)?;
|
||||
// A terminal timeout leaves no runtime owner for stop() to snapshot.
|
||||
// Checkpoint its consumed budget before publishing terminal status;
|
||||
// never refund time if an earlier checkpoint is already stricter.
|
||||
@@ -1033,14 +1259,13 @@ impl RootHealRecovery {
|
||||
(previous, remaining) => previous.or(remaining),
|
||||
};
|
||||
intent.retry_attempts = intent.retry_attempts.max(task.retry_attempts);
|
||||
let bytes = serde_json::to_vec(&intent)
|
||||
.map_err(|error| Error::Other(format!("Serialize root heal recovery checkpoint: {error}")))?;
|
||||
let bytes = intent.encode()?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task.id)?,
|
||||
Some(expected),
|
||||
Some(bytes.into()),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// This prefix must not overlap either namespace understood by schema-1 readers.
|
||||
@@ -68,24 +67,7 @@ impl RootHealReport {
|
||||
}
|
||||
|
||||
pub(super) fn encode(&self) -> Result<EcstoreDiskBytes> {
|
||||
struct BoundedWriter(Vec<u8>);
|
||||
impl Write for BoundedWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > MAX_ROOT_REPORT_BYTES.saturating_sub(self.0.len()) {
|
||||
return Err(std::io::Error::other("root heal report exceeds its size limit"));
|
||||
}
|
||||
self.0.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut writer = BoundedWriter(Vec::new());
|
||||
serde_json::to_writer(&mut writer, self)
|
||||
.map_err(|error| Error::Serialization(format!("Serialize root heal report: {error}")))?;
|
||||
Ok(writer.0.into())
|
||||
encode_root_record(self, MAX_ROOT_REPORT_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ use tempfile::TempDir;
|
||||
pub(super) mod admin_overlap;
|
||||
mod root_recovery;
|
||||
use uuid::Uuid;
|
||||
mod auto_scan;
|
||||
mod running_mainline;
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
|
||||
@@ -5670,7 +5671,7 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
process_manager_queue_once(&manager).await;
|
||||
let explicit_status = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if let Ok(status @ HealTaskStatus::Failed { .. }) = manager.get_task_status(&explicit_id).await {
|
||||
if let Ok(status @ HealTaskStatus::Completed) = manager.get_task_status(&explicit_id).await {
|
||||
break status;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
@@ -5678,7 +5679,8 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
})
|
||||
.await
|
||||
.expect("explicit timeout request should finish without using the zero default");
|
||||
assert!(matches!(explicit_status, HealTaskStatus::Failed { .. }));
|
||||
// Generation-bound admin tasks reach the successful mock heal without a namespace preflight.
|
||||
assert_eq!(explicit_status, HealTaskStatus::Completed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage::HealListItem;
|
||||
use crate::heal::storage_api::{
|
||||
EcstoreEndpointServerPools as EndpointServerPools, EcstoreEndpoints as Endpoints, EcstorePoolEndpoints as PoolEndpoints,
|
||||
ecstore_init_local_disks as init_local_disks,
|
||||
};
|
||||
use std::{
|
||||
future::Future,
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
const SET: &str = "pool_0_set_0";
|
||||
const SCAN_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
struct ScannerStorage {
|
||||
survivor: DiskStore,
|
||||
identity: ReplacementTargetIdentity,
|
||||
inventory_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for ScannerStorage {
|
||||
async fn replacement_intent_disks(&self, set: &str) -> Result<Vec<DiskStore>> {
|
||||
assert_eq!(set, SET, "discovery must use the target's erasure set");
|
||||
self.inventory_calls.fetch_add(1, Ordering::SeqCst);
|
||||
// This disk is deliberately outside the node's local registry. The
|
||||
// mock supplies the same inventory boundary as a remote survivor,
|
||||
// while real disk I/O exercises durable intent discovery and loading.
|
||||
Ok(vec![self.survivor.clone()])
|
||||
}
|
||||
|
||||
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
assert_eq!(targets, std::slice::from_ref(&self.identity.endpoint));
|
||||
Ok(vec![self.identity.clone()])
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Err(Error::other("scanner admission must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
|
||||
Err(Error::other("scanner admission must not rebuild objects"))
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, _: &str) -> Result<Option<BucketInfo>> {
|
||||
Err(Error::other("scanner admission must not inspect buckets"))
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Err(Error::other("same-incarnation discovery must reuse durable buckets"))
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
Err(Error::other("scanner admission must not start admin discovery"))
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
|
||||
Err(Error::other("scanner admission must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("scanner admission must not execute object healing"))
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
|
||||
Err(Error::other("scanner admission must not execute bucket healing"))
|
||||
}
|
||||
|
||||
async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("scanner admission must not execute format healing"))
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Err(Error::other("scanner admission must not traverse objects"))
|
||||
}
|
||||
|
||||
async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
|
||||
Err(Error::other("scanner admission must retain the durable survivor"))
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerFixture {
|
||||
_temp: TempDir,
|
||||
manager: HealManager,
|
||||
storage: Arc<ScannerStorage>,
|
||||
owner: ResumeManager,
|
||||
task_id: String,
|
||||
local_targets: Vec<String>,
|
||||
}
|
||||
|
||||
impl Drop for ScannerFixture {
|
||||
fn drop(&mut self) {
|
||||
self.manager.cancel_token.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl ScannerFixture {
|
||||
async fn new(unformatted_siblings: bool) -> Self {
|
||||
let temp = TempDir::new().expect("scanner replacement fixture directory");
|
||||
let disk_ids = (0..3).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let mut endpoints = Vec::new();
|
||||
for slot in 0..3 {
|
||||
let path = temp.path().join(format!("target-{slot}"));
|
||||
let metadata_path = path.join(crate::heal::RUSTFS_META_BUCKET);
|
||||
std::fs::create_dir_all(&metadata_path).expect("scanner local target metadata directory");
|
||||
if !unformatted_siblings || slot == 0 {
|
||||
let format = serde_json::json!({
|
||||
"version": "1",
|
||||
"format": "xl",
|
||||
"id": deployment_id,
|
||||
"xl": {
|
||||
"version": "3",
|
||||
"this": disk_ids[slot],
|
||||
"sets": [disk_ids],
|
||||
"distributionAlgo": "SIPMOD+PARITY"
|
||||
}
|
||||
});
|
||||
std::fs::write(
|
||||
metadata_path.join("format.json"),
|
||||
serde_json::to_vec(&format).expect("scanner format fixture serialization"),
|
||||
)
|
||||
.expect("scanner formatted target fixture");
|
||||
}
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(path.to_str().expect("scanner target path is UTF-8")).expect("scanner local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(slot);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let local_targets = endpoints.iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
init_local_disks(EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "replacement-scanner-recovery-test".into(),
|
||||
platform: "test".into(),
|
||||
}]))
|
||||
.await
|
||||
.expect("scanner local registry initialization");
|
||||
|
||||
let survivor = make_manager_resume_disk(&temp, "survivor").await;
|
||||
let local_disks = crate::heal::local_disk_map_read().await;
|
||||
assert_eq!(local_disks.len(), 3);
|
||||
assert!(
|
||||
!local_disks.contains_key(&survivor.endpoint().to_string()),
|
||||
"the durable anchor must be discoverable only through the set inventory"
|
||||
);
|
||||
for disk in local_disks.values().flatten() {
|
||||
let is_owner = disk.endpoint().to_string() == local_targets[0];
|
||||
match disk.get_disk_id().await {
|
||||
Ok(Some(id)) => assert!(!id.is_nil()),
|
||||
Err(DiskError::UnformattedDisk) => assert!(unformatted_siblings && !is_owner),
|
||||
result => panic!("unexpected fixture disk identity: {result:?}"),
|
||||
}
|
||||
assert_eq!(disk.runtime_state().as_str(), "online");
|
||||
}
|
||||
drop(local_disks);
|
||||
|
||||
let identity = ReplacementTargetIdentity {
|
||||
endpoint: local_targets[0].clone(),
|
||||
canonical_path: local_targets[0].clone(),
|
||||
physical_device_ids: vec!["replacement-target-0".into()],
|
||||
filesystem_identity: "replacement-target-0-incarnation".into(),
|
||||
};
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let owner = ResumeManager::new_replacement_intent(
|
||||
survivor.clone(),
|
||||
task_id.clone(),
|
||||
SET.into(),
|
||||
vec!["seeded-bucket".into()],
|
||||
vec![local_targets[0].clone()],
|
||||
vec![identity.clone()],
|
||||
)
|
||||
.await
|
||||
.expect("durable remote-survivor replacement intent");
|
||||
owner
|
||||
.mark_replacement_rebuilding(vec![identity.clone()])
|
||||
.await
|
||||
.expect("durable replacement rebuild phase");
|
||||
owner
|
||||
.record_replacement_failure(&Error::Storage(EcstoreError::FileNotFound), 1)
|
||||
.await
|
||||
.expect("durable in-budget FileNotFound failure");
|
||||
let storage = Arc::new(ScannerStorage {
|
||||
survivor,
|
||||
identity,
|
||||
inventory_calls: AtomicUsize::new(0),
|
||||
});
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
heal_interval: SCAN_INTERVAL,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
Self {
|
||||
_temp: temp,
|
||||
manager,
|
||||
storage,
|
||||
owner,
|
||||
task_id,
|
||||
local_targets,
|
||||
}
|
||||
}
|
||||
|
||||
async fn queued_request(&self) -> HealRequest {
|
||||
// Keep the paused clock from advancing while filesystem workers
|
||||
// finish. A wall-clock deadline also bounds a broken admission path.
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let queue = self.manager.heal_queue.lock().await;
|
||||
if let Some(item) = queue.heap.peek() {
|
||||
assert_eq!(queue.len(), 1, "one set must have exactly one admitted owner");
|
||||
return item.request.clone();
|
||||
}
|
||||
drop(queue);
|
||||
assert!(std::time::Instant::now() < deadline, "scanner tick did not admit replacement recovery");
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_original_owner_is_queued(&self) {
|
||||
let request = self.queued_request().await;
|
||||
assert_eq!(request.id, self.task_id, "scanner must replay the durable UUID");
|
||||
assert_eq!(request.heal_endpoints, vec![self.local_targets[0].clone()]);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
assert!(matches!(
|
||||
request.heal_type,
|
||||
HealType::ErasureSet { set_disk_id, buckets }
|
||||
if set_disk_id == SET && buckets == ["seeded-bucket"]
|
||||
));
|
||||
assert_eq!(
|
||||
self.manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("scanner anchor registry")
|
||||
.get(&self.task_id),
|
||||
Some(&self.storage.survivor.endpoint().to_string())
|
||||
);
|
||||
assert!(!self.manager.replacement_recovery_set_is_blocked(SET));
|
||||
assert_eq!(
|
||||
self.storage.inventory_calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"three local slots in one set must share one inventory lookup per tick"
|
||||
);
|
||||
let state = self.owner.get_state().await;
|
||||
assert_eq!(state.retry_count, 1, "discovery must not reset the durable retry budget");
|
||||
assert_eq!(state.replacement_targets, request.heal_endpoints);
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_without_advancing_clock<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::pin!(future);
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut future => return result,
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial(replacement_auto_scan)]
|
||||
async fn scanner_tick_recovers_remote_survivor_after_all_local_targets_are_formatted() {
|
||||
let fixture = ScannerFixture::new(false).await;
|
||||
fixture
|
||||
.manager
|
||||
.start_auto_disk_scanner()
|
||||
.await
|
||||
.expect("start only the periodic replacement scanner");
|
||||
fixture.assert_original_owner_is_queued().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial(replacement_auto_scan)]
|
||||
async fn scanner_tick_finishes_existing_owner_before_admitting_new_unformatted_targets() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
|
||||
let fixture = ScannerFixture::new(true).await;
|
||||
fixture
|
||||
.manager
|
||||
.start_auto_disk_scanner()
|
||||
.await
|
||||
.expect("start only the periodic replacement scanner");
|
||||
fixture.assert_original_owner_is_queued().await;
|
||||
|
||||
// The real task's verified completion publishes the proof and
|
||||
// removes its live intent before sibling admission is allowed.
|
||||
let owner_removed = finish_without_advancing_clock(async {
|
||||
fixture
|
||||
.owner
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("durable owner completion proof");
|
||||
fixture
|
||||
.owner
|
||||
.mark_replacement_cleanup_pending()
|
||||
.await
|
||||
.expect("durable owner cleanup phase");
|
||||
fixture
|
||||
.owner
|
||||
.cleanup()
|
||||
.await
|
||||
.expect("completed owner releases its durable intent");
|
||||
!ResumeManager::has_replacement_intent(&fixture.storage.survivor, &fixture.task_id).await
|
||||
})
|
||||
.await;
|
||||
assert!(owner_removed, "completed ownership must be removed before the next generation");
|
||||
let finished = fixture
|
||||
.manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.pop_next()
|
||||
.expect("remove the completed queued owner");
|
||||
assert_eq!(finished.id, fixture.task_id);
|
||||
tokio::time::advance(SCAN_INTERVAL).await;
|
||||
|
||||
let next = fixture.queued_request().await;
|
||||
assert_ne!(next.id, fixture.task_id, "the follow-up replacement gets a fresh generation");
|
||||
assert_eq!(next.heal_endpoints, fixture.local_targets[1..]);
|
||||
assert_eq!(next.source, HealRequestSource::AutoHeal);
|
||||
assert!(!fixture.manager.replacement_recovery_set_is_blocked(SET));
|
||||
assert_eq!(fixture.storage.inventory_calls.load(Ordering::SeqCst), 2);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_disk_index_keeps_first_snapshot_and_sorted_unique_endpoints() {
|
||||
let temp = TempDir::new().expect("recovery disk index fixture");
|
||||
let local = make_manager_resume_disk(&temp, "a-local").await;
|
||||
let duplicate = make_manager_resume_disk(&temp, "a-local").await;
|
||||
let survivor = make_manager_resume_disk(&temp, "z-survivor").await;
|
||||
assert!(!Arc::ptr_eq(&local, &duplicate), "duplicate must be a different disk snapshot");
|
||||
let local_endpoint = local.endpoint().to_string();
|
||||
let survivor_endpoint = survivor.endpoint().to_string();
|
||||
let mut inventory = std::collections::BTreeMap::new();
|
||||
super::super::auto_scan::extend_replacement_recovery_disks(&mut inventory, [local.clone()]);
|
||||
super::super::auto_scan::extend_replacement_recovery_disks(&mut inventory, [survivor.clone(), duplicate, survivor]);
|
||||
assert_eq!(inventory.len(), 2, "each endpoint is discovered once");
|
||||
assert!(
|
||||
Arc::ptr_eq(inventory.get(&local_endpoint).expect("local endpoint remains indexed"), &local),
|
||||
"later set inventories must not replace the local snapshot"
|
||||
);
|
||||
assert_eq!(
|
||||
inventory.keys().collect::<Vec<_>>(),
|
||||
vec![&local_endpoint, &survivor_endpoint],
|
||||
"discovery order remains lexical regardless of insertion order"
|
||||
);
|
||||
}
|
||||
@@ -1844,7 +1844,7 @@ async fn root_recovery_queued_non_root_admin_owner_is_not_priority_displaced() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_legacy_schema_replays_as_cluster() {
|
||||
async fn root_recovery_legacy_cluster_requires_a_fresh_generation_bound_task() {
|
||||
#[derive(serde::Serialize)]
|
||||
struct LegacyRootHealIntent<'a> {
|
||||
schema: u32,
|
||||
@@ -1873,10 +1873,113 @@ async fn root_recovery_legacy_schema_replays_as_cluster() {
|
||||
|
||||
let restarted = recovery_manager(vec![disk]);
|
||||
restarted.replay_root_heals().await.expect("replay legacy root record");
|
||||
let queued = restarted.heal_queue.lock().await.requests().cloned().collect::<Vec<_>>();
|
||||
assert_eq!(queued.len(), 1);
|
||||
assert_eq!(queued[0].id, request.id);
|
||||
assert_eq!(queued[0].heal_type, HealType::Cluster);
|
||||
assert!(
|
||||
restarted.heal_queue.lock().await.requests().next().is_none(),
|
||||
"unfenced legacy work must not bind to a later bucket generation"
|
||||
);
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.inventory()
|
||||
.await
|
||||
.expect("quarantined legacy task")
|
||||
.quarantined_task_ids
|
||||
.contains(&request.id)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_scope_preserves_generations_and_completed_prefix_on_handoff() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
use std::collections::BTreeMap;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let mut scope = AdminClusterScope {
|
||||
buckets: BTreeMap::from([
|
||||
("alpha-bucket".to_owned(), Uuid::from_u128(42)),
|
||||
("bravo-bucket".to_owned(), Uuid::from_u128(43)),
|
||||
]),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("freeze bucket generations before mutation");
|
||||
scope.next_bucket_index = 1;
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("checkpoint completed empty bucket");
|
||||
let retry = task.retry_request();
|
||||
manager
|
||||
.root_recovery
|
||||
.persist(&retry)
|
||||
.await
|
||||
.expect("shutdown/retry handoff retains scope");
|
||||
let mut changed_options = retry.clone();
|
||||
changed_options.options.recursive = !changed_options.options.recursive;
|
||||
assert!(
|
||||
manager.root_recovery.persist(&changed_options).await.is_err(),
|
||||
"handoff cannot reinterpret completed work"
|
||||
);
|
||||
drop(manager);
|
||||
let restarted = recovery_manager(vec![disk]);
|
||||
let replay = HealTask::from_request(retry, Arc::new(MockStorage));
|
||||
assert_eq!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.cluster_scope(&replay)
|
||||
.await
|
||||
.expect("read frozen scope"),
|
||||
Some(scope.clone())
|
||||
);
|
||||
let mut changed = scope.clone();
|
||||
changed.buckets.insert("alpha-bucket".to_owned(), Uuid::from_u128(44));
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"never rebind a saved generation"
|
||||
);
|
||||
changed = scope.clone();
|
||||
changed.next_bucket_index = 0;
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"never rewind a committed prefix"
|
||||
);
|
||||
changed = scope.clone();
|
||||
changed.buckets.insert("../invalid".to_owned(), Uuid::from_u128(45));
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"reject decoded path aliases"
|
||||
);
|
||||
assert_eq!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.cluster_scope(&replay)
|
||||
.await
|
||||
.expect("failed updates preserve scope"),
|
||||
Some(scope)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1899,6 +2002,155 @@ async fn root_recovery_rejected_admin_start_does_not_persist() {
|
||||
assert_eq!(restarted.get_queue_length().await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_replay_rejects_successor_and_preserves_completed_scope() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
use crate::heal::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome};
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let bucket = format!("pinned-{}", Uuid::new_v4().simple());
|
||||
let expected = Uuid::from_u128(42);
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let mut scope = AdminClusterScope {
|
||||
buckets: std::collections::BTreeMap::from([(bucket.clone(), expected)]),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("freeze generation");
|
||||
bucket_incarnations()
|
||||
.lock()
|
||||
.expect("generation fixture")
|
||||
.insert(bucket.clone(), Some(Uuid::from_u128(99)));
|
||||
let replay =
|
||||
HealTask::from_request(request.clone(), Arc::new(MockStorage)).with_admin_recovery(manager.root_recovery.clone());
|
||||
assert!(
|
||||
matches!(replay.execute().await, Err(Error::StaleBucketIncarnation { .. })),
|
||||
"replay must reject a successor before traversal"
|
||||
);
|
||||
assert_eq!(replay.get_outcome().await.counters.processed, 0);
|
||||
bucket_incarnations()
|
||||
.lock()
|
||||
.expect("generation fixture")
|
||||
.insert(bucket.clone(), Some(expected));
|
||||
scope.next_bucket_index = 1;
|
||||
scope.outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.clone(),
|
||||
object: "completed-object".to_owned(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(expected),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
});
|
||||
scope.progress.objects_scanned = 1;
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("completed bucket with retained outcome");
|
||||
let replay = HealTask::from_request(request, Arc::new(MockStorage)).with_admin_recovery(manager.root_recovery.clone());
|
||||
replay
|
||||
.execute()
|
||||
.await
|
||||
.expect("completed frozen scope needs no rediscovery or repair");
|
||||
assert_eq!(replay.get_outcome().await.coverage, crate::heal::outcome::HealTraversalCoverage::Complete);
|
||||
assert_eq!(replay.get_outcome().await.counters.processed, 1);
|
||||
assert_eq!(replay.get_progress().await.objects_scanned, 1);
|
||||
bucket_incarnations().lock().expect("generation fixture").remove(&bucket);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_inventory_exceeds_legacy_limit_without_losing_bindings() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let scope = AdminClusterScope {
|
||||
buckets: (0..1500)
|
||||
.map(|index| (format!("bucket-{index:05}"), Uuid::from_u128(42)))
|
||||
.collect(),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
assert!(scope.checkpoint_interval() > 1, "large inventories must batch durable rewrites");
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("persist large frozen inventory");
|
||||
let path = format!("root-heal-{}.json", request.id);
|
||||
let bytes = disk.read_all(RUSTFS_META_BUCKET, &path).await.expect("scope bytes");
|
||||
assert!(bytes.len() > MAX_ROOT_RECOVERY_BYTES, "fixture must cross legacy size limit");
|
||||
assert_eq!(
|
||||
manager
|
||||
.root_recovery
|
||||
.cluster_scope(&task)
|
||||
.await
|
||||
.expect("all bindings survive"),
|
||||
Some(scope)
|
||||
);
|
||||
let json = String::from_utf8(bytes.to_vec()).expect("UTF-8 scope");
|
||||
let binding = format!("\"bucket-00000\":\"{}\"", Uuid::from_u128(42));
|
||||
let duplicate = json.replacen(&binding, &format!("{binding},{binding}"), 1);
|
||||
assert_ne!(duplicate, json, "inject a duplicate map key");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, duplicate.into_bytes().into())
|
||||
.await
|
||||
.expect("inject corrupt duplicate binding");
|
||||
assert!(
|
||||
manager.root_recovery.cluster_scope(&task).await.is_err(),
|
||||
"duplicate identity bindings fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_rejects_inventory_without_checkpoint_headroom() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request, Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let scope = AdminClusterScope {
|
||||
// This inventory fits the 16 MiB record limit but leaves too little
|
||||
// space for future object outcomes. Reject it before authorizing repair.
|
||||
buckets: (0..152_000)
|
||||
.map(|index| (format!("bucket-{index:056}"), Uuid::from_u128(42)))
|
||||
.collect(),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
assert!(manager.root_recovery.checkpoint_cluster_scope(&task, &scope).await.is_err());
|
||||
assert!(
|
||||
manager
|
||||
.root_recovery
|
||||
.cluster_scope(&task)
|
||||
.await
|
||||
.expect("original admission survives")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_queued_owner_is_not_priority_displaced() {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
@@ -2127,7 +2379,16 @@ async fn root_recovery_force_start_replaces_fresh_queued_and_retrying_admin_root
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay() {
|
||||
for kind in ["truncated", "schema", "identity", "option", "no_lock", "oversized"] {
|
||||
use crate::heal::manager::root_recovery::MAX_CLUSTER_RECOVERY_BYTES;
|
||||
for kind in [
|
||||
"truncated",
|
||||
"schema",
|
||||
"identity",
|
||||
"option",
|
||||
"no_lock",
|
||||
"unscoped_oversized",
|
||||
"oversized",
|
||||
] {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let valid = root_request();
|
||||
@@ -2142,7 +2403,7 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
let original = disk.read_all(RUSTFS_META_BUCKET, &path).await.expect("read root record");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&original).expect("record JSON");
|
||||
match kind {
|
||||
"schema" => value["schema"] = 4.into(),
|
||||
"schema" => value["schema"] = u32::MAX.into(),
|
||||
"identity" => value["task_id"] = valid.id.clone().into(),
|
||||
"option" => value["options"]["future_delete_mode"] = true.into(),
|
||||
"no_lock" => value["options"]["no_lock"] = true.into(),
|
||||
@@ -2150,7 +2411,12 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
}
|
||||
let bytes = match kind {
|
||||
"truncated" => b"{".to_vec(),
|
||||
"oversized" => vec![b' '; MAX_ROOT_RECOVERY_BYTES + 1],
|
||||
"oversized" => vec![b' '; MAX_CLUSTER_RECOVERY_BYTES + 1],
|
||||
"unscoped_oversized" => {
|
||||
let mut bytes = serde_json::to_vec(&value).expect("valid unscoped record");
|
||||
bytes.resize(MAX_ROOT_RECOVERY_BYTES + 1, b' ');
|
||||
bytes
|
||||
}
|
||||
_ => serde_json::to_vec(&value).expect("modified record"),
|
||||
};
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.clone().into())
|
||||
@@ -2173,7 +2439,11 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
let marker: serde_json::Value = serde_json::from_slice(&marker).expect("quarantine marker JSON");
|
||||
assert_eq!(marker["task_id"], invalid.id, "kind={kind}");
|
||||
assert_eq!(marker["record_kind"], "intent", "kind={kind}");
|
||||
assert_eq!(marker["source_prefix_len"], bytes.len().min(MAX_ROOT_RECOVERY_BYTES + 1), "kind={kind}");
|
||||
assert_eq!(
|
||||
marker["source_prefix_len"],
|
||||
bytes.len().min(MAX_CLUSTER_RECOVERY_BYTES + 1),
|
||||
"kind={kind}"
|
||||
);
|
||||
assert_eq!(marker["source_oversized"], kind == "oversized", "kind={kind}");
|
||||
|
||||
let mut reused = invalid.clone();
|
||||
@@ -2195,6 +2465,51 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_legacy_oversized_quarantine_survives_larger_intent_limit() {
|
||||
use sha2::{Digest, Sha256};
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
let path = format!("root-heal-{}.json", request.id);
|
||||
let mut bytes = vec![b' '; MAX_ROOT_RECOVERY_BYTES + 100];
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.clone().into())
|
||||
.await
|
||||
.expect("old oversized intent");
|
||||
let marker = serde_json::json!({
|
||||
"schema": 1,
|
||||
"task_id": request.id,
|
||||
"record_kind": "intent",
|
||||
"source_prefix_len": MAX_ROOT_RECOVERY_BYTES + 1,
|
||||
"source_prefix_sha256": Sha256::digest(&bytes[..MAX_ROOT_RECOVERY_BYTES + 1]).to_vec(),
|
||||
"source_oversized": true,
|
||||
"reason": "legacy size limit",
|
||||
"quarantined_at": SystemTime::now(),
|
||||
});
|
||||
disk.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("quarantined-root-heal-intent-{}.json", request.id),
|
||||
serde_json::to_vec(&marker).unwrap().into(),
|
||||
)
|
||||
.await
|
||||
.expect("old quarantine marker");
|
||||
let inventory = manager
|
||||
.root_recovery
|
||||
.inventory()
|
||||
.await
|
||||
.expect("upgrade preserves old quarantine");
|
||||
assert!(inventory.requests.is_empty());
|
||||
assert!(inventory.quarantined_task_ids.contains(&request.id));
|
||||
bytes[0] = b'{';
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.into())
|
||||
.await
|
||||
.expect("alter retained prefix");
|
||||
assert!(
|
||||
manager.root_recovery.inventory().await.is_err(),
|
||||
"a changed legacy source still fails closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_quarantine_source_change_fails_closed() {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::capacity::is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket;
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME,
|
||||
load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached,
|
||||
@@ -26,7 +27,13 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::layout::{
|
||||
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks as ecstore_init_local_disks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
HealObjectStorageResult as EcstoreHealObjectStorageResult, POOL_META_NAME as ECSTORE_POOL_META_NAME,
|
||||
@@ -40,8 +47,8 @@ pub(crate) mod owner {
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached,
|
||||
ecstore_local_disk_map_read,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_is_reserved_or_invalid_bucket,
|
||||
ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
@@ -132,6 +132,19 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<()> {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, self.bucket_incarnation_id, opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
self.pace_mainline().await?;
|
||||
if self.source == HealRequestSource::Admin && matches!(self.heal_type, HealType::Bucket { .. }) {
|
||||
@@ -169,8 +182,10 @@ impl HealTask {
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let bucket_exists = self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_some();
|
||||
if !bucket_exists {
|
||||
// Bound administrator repair validates existence and generation under the
|
||||
// lifecycle fence. Ordinary read quorum can reject recoverable directory loss.
|
||||
let bound_admin_repair = self.source == HealRequestSource::Admin && self.bucket_incarnation_id.is_some();
|
||||
if !bound_admin_repair && self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_none() {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
@@ -224,18 +239,11 @@ impl HealTask {
|
||||
remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted,
|
||||
..heal_opts
|
||||
};
|
||||
if let Some(result) = self
|
||||
.await_with_control(self.storage.recover_delete_transactions(
|
||||
bucket,
|
||||
None,
|
||||
self.bucket_incarnation_id,
|
||||
&recovery_opts,
|
||||
))
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
// Removal must settle survivor journals before deleting an empty
|
||||
// replica's publication directory; repair must establish it first.
|
||||
if heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
}
|
||||
|
||||
let heal_result = match self.bucket_incarnation_id {
|
||||
Some(expected) => {
|
||||
self.await_with_control(self.storage.heal_bucket_at_incarnation(bucket, expected, &heal_opts))
|
||||
@@ -261,6 +269,10 @@ impl HealTask {
|
||||
);
|
||||
self.record_result_item(result).await;
|
||||
|
||||
if !heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
}
|
||||
|
||||
if self.options.recursive {
|
||||
self.heal_bucket_objects(bucket, "").await?;
|
||||
}
|
||||
@@ -308,22 +320,76 @@ impl HealTask {
|
||||
"Heal cluster started"
|
||||
);
|
||||
|
||||
let bucket_infos = if self.source == HealRequestSource::Admin {
|
||||
self.await_with_control(self.storage.list_buckets_for_admin_heal(None))
|
||||
.await?
|
||||
let mut cluster_scope = if self.source == HealRequestSource::Admin && !self.options.dry_run {
|
||||
let saved = if let Some(recovery) = &self.admin_recovery {
|
||||
recovery.cluster_scope(self).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let scope = if let Some(scope) = saved {
|
||||
scope
|
||||
} else {
|
||||
let mut buckets = std::collections::BTreeMap::new();
|
||||
for bucket in self
|
||||
.await_with_control(self.storage.list_buckets_for_admin_heal(None))
|
||||
.await?
|
||||
{
|
||||
let incarnation = self
|
||||
.await_with_control(self.storage.admit_bucket_incarnation(&bucket.name))
|
||||
.await?;
|
||||
buckets.insert(bucket.name, incarnation);
|
||||
}
|
||||
let scope = super::super::manager::root_recovery::AdminClusterScope {
|
||||
buckets,
|
||||
next_bucket_index: 0,
|
||||
outcome: self.get_outcome().await,
|
||||
progress: self.progress.read().await.clone(),
|
||||
};
|
||||
if let Some(recovery) = &self.admin_recovery {
|
||||
recovery.checkpoint_cluster_scope(self, &scope).await?;
|
||||
}
|
||||
scope
|
||||
};
|
||||
// Validate the frozen generations on every replay, including completed buckets.
|
||||
for (bucket, incarnation) in &scope.buckets {
|
||||
self.await_with_control(self.storage.validate_bucket_incarnation(bucket, Some(*incarnation)))
|
||||
.await?;
|
||||
}
|
||||
self.restore_outcome(scope.outcome.clone()).await;
|
||||
*self.progress.write().await = scope.progress.clone();
|
||||
Some(scope)
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_buckets()).await?
|
||||
None
|
||||
};
|
||||
let buckets = if let Some(scope) = &cluster_scope {
|
||||
scope
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|(name, incarnation)| (name.clone(), Some(*incarnation)))
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
let listed = if self.source == HealRequestSource::Admin {
|
||||
self.await_with_control(self.storage.list_buckets_for_admin_heal(None))
|
||||
.await?
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_buckets()).await?
|
||||
};
|
||||
listed.into_iter().map(|bucket| (bucket.name, None)).collect()
|
||||
};
|
||||
let mut failed = 0_u64;
|
||||
let mut retryable = 0_u64;
|
||||
let mut permanent = 0_u64;
|
||||
let mut first_object = None;
|
||||
let mut first_error = None;
|
||||
for bucket_info in bucket_infos {
|
||||
let start = cluster_scope.as_ref().map_or(0, |scope| scope.next_bucket_index);
|
||||
let mut checkpointed = start;
|
||||
for (index, (bucket, incarnation)) in buckets.into_iter().enumerate().skip(start) {
|
||||
self.check_control_flags().await?;
|
||||
let mut bucket_task = self.clone();
|
||||
bucket_task.bucket_incarnation_id = incarnation;
|
||||
let mut retry_attempt = 0_u32;
|
||||
loop {
|
||||
match self.heal_bucket(&bucket_info.name).await {
|
||||
match bucket_task.heal_bucket(&bucket).await {
|
||||
Ok(()) => break,
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
@@ -352,15 +418,37 @@ impl HealTask {
|
||||
} else {
|
||||
permanent = permanent.saturating_add(1);
|
||||
}
|
||||
first_object.get_or_insert(bucket_info.name.clone());
|
||||
first_object.get_or_insert(bucket.clone());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Retain only the successful contiguous prefix. A later retry replays
|
||||
// any work after a failure without inheriting its transient counters.
|
||||
if failed == 0
|
||||
&& let Some(scope) = &mut cluster_scope
|
||||
{
|
||||
scope.next_bucket_index = index.saturating_add(1);
|
||||
scope.outcome = self.get_outcome().await;
|
||||
scope.progress = self.progress.read().await.clone();
|
||||
if (scope.next_bucket_index.saturating_sub(checkpointed) >= scope.checkpoint_interval()
|
||||
|| scope.next_bucket_index == scope.buckets.len())
|
||||
&& let Some(recovery) = &self.admin_recovery
|
||||
{
|
||||
recovery.checkpoint_cluster_scope(self, scope).await?;
|
||||
checkpointed = scope.next_bucket_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
if let Some(scope) = &cluster_scope
|
||||
&& scope.next_bucket_index > checkpointed
|
||||
&& let Some(recovery) = &self.admin_recovery
|
||||
{
|
||||
recovery.checkpoint_cluster_scope(self, scope).await?;
|
||||
}
|
||||
let failure = BatchHealFailure {
|
||||
scope: "cluster".to_string(),
|
||||
failed,
|
||||
|
||||
@@ -460,21 +460,6 @@ impl HealTask {
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let recovery_opts = HealOpts {
|
||||
remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted,
|
||||
..bucket_heal_opts
|
||||
};
|
||||
// Include metadata-less transactions for both startup and admin heals.
|
||||
for bucket in &buckets {
|
||||
self.check_control_flags().await?;
|
||||
if let Some(result) = self
|
||||
.await_with_control(self.storage.recover_delete_transactions(bucket, None, None, &recovery_opts))
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) {
|
||||
// Check control flags before starting each bucket heal
|
||||
self.check_control_flags().await?;
|
||||
@@ -505,6 +490,16 @@ impl HealTask {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
// Adopt delete proofs only after their publication bucket exists.
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, None, &bucket_heal_opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Create erasure set healer with resume support
|
||||
|
||||
@@ -102,6 +102,10 @@ impl HealStorageAPI for PausedTraversal {
|
||||
self.inner.admit_bucket_incarnation(bucket).await
|
||||
}
|
||||
|
||||
async fn heal_bucket_at_incarnation(&self, bucket: &str, expected: Uuid, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
self.inner.heal_bucket_at_incarnation(bucket, expected, opts).await
|
||||
}
|
||||
|
||||
async fn heal_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -17,10 +17,11 @@
|
||||
use http::HeaderMap;
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
outcome::{HealExecutionOutcome, HealTraversalCoverage},
|
||||
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
@@ -33,7 +34,7 @@ use walkdir::WalkDir;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};
|
||||
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _, WriteCompletion};
|
||||
|
||||
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
|
||||
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
|
||||
@@ -107,16 +108,183 @@ async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
|
||||
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
|
||||
let mut reader = PutObjReader::from_vec(data.to_vec());
|
||||
let object_info = (**ecstore)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Failed to upload test object");
|
||||
|
||||
info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
|
||||
}
|
||||
|
||||
async fn empty_replacement_with_committed_delete(
|
||||
ecstore: &Arc<ECStore>,
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> (uuid::Uuid, uuid::Uuid) {
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use storage_api::integration::{
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryState, DiskAPI as _, DiskSetSelector, StorageAdminApi as _,
|
||||
};
|
||||
assert_eq!(disk_paths.len(), 4, "replacement fixture needs four paths");
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("admit the survivor bucket generation");
|
||||
let disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("read the live fixture set");
|
||||
assert_eq!(disks.len(), 4);
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
// Retain a certified survivor decision without the normal delete's
|
||||
// immediate recovery/cleanup consuming the fault input.
|
||||
for disk in &disks[1..] {
|
||||
let disk = disk.as_ref().expect("all three preparation survivors are online");
|
||||
disk.delete_version_recoverable(
|
||||
bucket,
|
||||
object,
|
||||
FileInfo {
|
||||
name: object.to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
delete_recovery_bucket_incarnation: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prepare the same delete on every survivor");
|
||||
}
|
||||
for disk in &disks[1..] {
|
||||
let disk = disk.as_ref().expect("all three survivors are online");
|
||||
let prepared = disk
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await
|
||||
.expect("inspect prepared survivor decision");
|
||||
assert_eq!(prepared.state, DeleteRecoveryState::NewPrepared);
|
||||
let committed = disk
|
||||
.settle_delete_transaction(bucket, object, transaction, prepared.fingerprint, DeleteRecoveryAction::PublishCommit)
|
||||
.await
|
||||
.expect("publish the retained survivor decision");
|
||||
assert_eq!(committed.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
// Linux disk I/O pins the root inode. Clearing its contents models the
|
||||
// empty target on every platform; renaming it leaves Linux handles on
|
||||
// the retired data and does not simulate a replacement for this test.
|
||||
for entry in std::fs::read_dir(&disk_paths[0]).expect("read target root") {
|
||||
let entry = entry.expect("read target entry");
|
||||
if entry.file_type().expect("target entry type").is_dir() {
|
||||
std::fs::remove_dir_all(entry.path()).expect("clear target directory");
|
||||
} else {
|
||||
std::fs::remove_file(entry.path()).expect("clear target file");
|
||||
}
|
||||
}
|
||||
(incarnation, transaction)
|
||||
}
|
||||
|
||||
mod serial_tests {
|
||||
use super::*;
|
||||
|
||||
async fn bucket_directory_loss_task(cluster: bool, missing: usize, dry_run: bool, source: HealRequestSource) {
|
||||
let temp = tempfile::tempdir().expect("isolated bucket recovery fixture");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "task-directory-recovery";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let expected = storage
|
||||
.admit_bucket_incarnation(bucket)
|
||||
.await
|
||||
.expect("committed bucket generation");
|
||||
for disk in env.disk_paths.iter().rev().take(missing) {
|
||||
fs::remove_dir(disk.join(bucket))
|
||||
.await
|
||||
.expect("remove only an empty bucket directory");
|
||||
}
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).exists()).count(), 4 - missing);
|
||||
let mut request = HealRequest::new(
|
||||
if cluster {
|
||||
HealType::Cluster
|
||||
} else {
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
}
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run,
|
||||
recreate_missing: false,
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = source;
|
||||
if !cluster && source == HealRequestSource::Admin {
|
||||
request.bucket_incarnation_id = Some(expected);
|
||||
}
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
let result = task.execute().await;
|
||||
if missing == 4 || source != HealRequestSource::Admin {
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"unproven or unbound recovery must fail: missing={missing}, source={source:?}"
|
||||
);
|
||||
} else {
|
||||
result.unwrap_or_else(|error| {
|
||||
panic!(
|
||||
"administrator task must reach generation-bound reconstruction: cluster={cluster}, missing={missing}: {error}"
|
||||
)
|
||||
});
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
}
|
||||
let remaining = if dry_run || missing == 4 || source != HealRequestSource::Admin {
|
||||
4 - missing
|
||||
} else {
|
||||
4
|
||||
};
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).is_dir()).count(), remaining);
|
||||
assert_eq!(storage.admit_bucket_incarnation(bucket).await.expect("retained metadata"), expected);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_from_task_entry() {
|
||||
for cluster in [false, true] {
|
||||
// Two surviving directories satisfy ordinary reads; one does not.
|
||||
for missing in [2, 3] {
|
||||
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_dry_run_from_task_entry() {
|
||||
bucket_directory_loss_task(false, 3, true, HealRequestSource::Admin).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_rejects_unproven_scope() {
|
||||
bucket_directory_loss_task(false, 4, false, HealRequestSource::Admin).await;
|
||||
bucket_directory_loss_task(false, 3, false, HealRequestSource::Internal).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_object_basic() {
|
||||
@@ -533,3 +701,209 @@ mod serial_tests {
|
||||
info!("Direct heal storage API test passed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn replacement_delete_recovery_on_fresh_target() {
|
||||
// Match the debug server's worker stack for the composed Heal future.
|
||||
// Libtest otherwise polls it on its smaller calling-thread stack.
|
||||
const STACK_SIZE: usize = 8 * 1024 * 1024;
|
||||
std::thread::Builder::new()
|
||||
.name("replacement-delete-recovery".to_owned())
|
||||
.stack_size(STACK_SIZE)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(4)
|
||||
.thread_stack_size(STACK_SIZE)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("replacement recovery runtime");
|
||||
runtime.block_on(replacement_delete_recovery_on_fresh_target_case());
|
||||
})
|
||||
.expect("replacement recovery test thread")
|
||||
.join()
|
||||
.expect("replacement recovery test result");
|
||||
}
|
||||
|
||||
async fn replacement_delete_recovery_on_fresh_target_case() {
|
||||
use rustfs_heal::Error;
|
||||
use rustfs_heal::heal::task::HealTask;
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use storage_api::integration::{
|
||||
DeleteRecoveryState, DiskAPI as _, DiskError, DiskSetSelector, EcstoreStorageError, StorageAdminApi as _,
|
||||
};
|
||||
|
||||
for (source, erasure_set) in [
|
||||
(HealRequestSource::AutoHeal, true),
|
||||
(HealRequestSource::Admin, true),
|
||||
(HealRequestSource::Admin, false),
|
||||
] {
|
||||
let (disk_paths, ecstore, storage) = heal_env().await;
|
||||
let bucket = "replacement-delete-prepass";
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
let original = non_inline_test_data();
|
||||
for key in ["live", "deleted"] {
|
||||
upload_test_object(&ecstore, bucket, key, &original).await;
|
||||
wait_for_object_copies(&disk_paths, bucket, key).await;
|
||||
}
|
||||
let (incarnation, transaction) = empty_replacement_with_committed_delete(&ecstore, &disk_paths, bucket, "deleted").await;
|
||||
let updated = vec![17; NON_INLINE_TEST_DATA_SIZE];
|
||||
upload_test_object(&ecstore, bucket, "live", &updated).await;
|
||||
upload_test_object(&ecstore, bucket, "outage-created", &original).await;
|
||||
let (_, error) = storage.heal_format(false).await.expect("format target");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("survivor incarnation remains authoritative"),
|
||||
incarnation
|
||||
);
|
||||
let current_disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("read the formatted live set");
|
||||
assert!(
|
||||
matches!(
|
||||
current_disks[0]
|
||||
.as_ref()
|
||||
.expect("formatted target is attached")
|
||||
.stat_volume(bucket)
|
||||
.await,
|
||||
Err(DiskError::VolumeNotFound)
|
||||
),
|
||||
"the formatted target must still lack its publication bucket"
|
||||
);
|
||||
for disk in ¤t_disks[1..] {
|
||||
let retained = disk
|
||||
.as_ref()
|
||||
.expect("survivor remains attached")
|
||||
.inspect_delete_transaction(bucket, "deleted", transaction)
|
||||
.await
|
||||
.expect("retained committed survivor proof");
|
||||
assert_eq!(retained.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
// A committed survivor journal must be adopted, but publishing its
|
||||
// witness cannot recreate a missing bucket outside lifecycle ownership.
|
||||
let premature = storage
|
||||
.recover_delete_transactions(bucket, None, None, &HealOpts::default())
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&premature, Err(Error::Storage(EcstoreStorageError::FileNotFound))),
|
||||
"{source:?} erasure_set={erasure_set}: {premature:?}"
|
||||
);
|
||||
let heal_type = if erasure_set {
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![bucket.to_owned()],
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
}
|
||||
} else {
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
}
|
||||
};
|
||||
let mut request = HealRequest::new(
|
||||
heal_type,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = source;
|
||||
if !erasure_set {
|
||||
request.bucket_incarnation_id = Some(incarnation);
|
||||
}
|
||||
HealTask::from_request(request, storage.clone())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{source:?} erasure_set={erasure_set}: {err}"));
|
||||
// Check physical repair before GET can hide missing shards via read repair.
|
||||
for key in ["live", "outage-created"] {
|
||||
assert!(disk_paths[0].join(bucket).join(key).join("xl.meta").exists());
|
||||
assert!(find_part_file(&disk_paths[0].join(bucket).join(key)).is_some());
|
||||
}
|
||||
for (key, expected) in [("live", updated.as_slice()), ("outage-created", original.as_slice())] {
|
||||
let mut reader = ecstore
|
||||
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("open rebuilt object");
|
||||
assert_eq!(reader.read_all().await.expect("read rebuilt bytes"), expected);
|
||||
}
|
||||
assert!(matches!(
|
||||
ecstore.get_object_info(bucket, "deleted", &ObjectOptions::default()).await,
|
||||
Err(EcstoreStorageError::ObjectNotFound(_, _))
|
||||
));
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("retained incarnation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn bucket_removal_recovers_delete_proofs_before_removing_empty_volumes() {
|
||||
use rustfs_heal::heal::task::HealTask;
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use storage_api::integration::{DeleteRecoveryState, DiskAPI as _, DiskSetSelector, StorageAdminApi as _};
|
||||
let (disk_paths, ecstore, storage) = heal_env().await;
|
||||
let bucket = "delete-proof-before-volume-removal";
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
upload_test_object(&ecstore, bucket, "deleted", &non_inline_test_data()).await;
|
||||
wait_for_object_copies(&disk_paths, bucket, "deleted").await;
|
||||
let (incarnation, transaction) = empty_replacement_with_committed_delete(&ecstore, &disk_paths, bucket, "deleted").await;
|
||||
let (_, error) = storage.heal_format(false).await.expect("format target");
|
||||
assert!(error.is_none());
|
||||
storage
|
||||
.heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("empty target volume");
|
||||
assert!(disk_paths[0].join(bucket).exists());
|
||||
let disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("live removal fixture set");
|
||||
for disk in &disks[1..] {
|
||||
let retained = disk
|
||||
.as_ref()
|
||||
.expect("removal survivor remains attached")
|
||||
.inspect_delete_transaction(bucket, "deleted", transaction)
|
||||
.await
|
||||
.expect("removal must still adopt the retained delete proof");
|
||||
assert_eq!(retained.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: false,
|
||||
remove_corrupted: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
request.bucket_incarnation_id = Some(incarnation);
|
||||
HealTask::from_request(request, storage)
|
||||
.execute()
|
||||
.await
|
||||
.expect("settle journals before deleting empty volumes");
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| !path.join(bucket).exists()),
|
||||
"all empty replicas should be removed after proof settlement"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -21,7 +21,8 @@ pub(crate) mod endpoint_index {
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod integration {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk,
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryState, DiskAPI, DiskError, DiskOption, DiskStore, Endpoint,
|
||||
RUSTFS_META_BUCKET, ReadOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::error::StorageError as EcstoreStorageError;
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
|
||||
@@ -120,8 +120,11 @@ pub fn object_lock_put_integrity(
|
||||
/// stored bytes differ from what the source sent — e.g. a target that does not
|
||||
/// decode `aws-chunked` framing stores the frames verbatim and returns their
|
||||
/// ETag. Only a plain single-part MD5 ETag on both sides is decidable; a
|
||||
/// multipart or opaque (encrypted) ETag, or a withheld replica ETag, returns
|
||||
/// `false` because no corruption can be concluded from it.
|
||||
/// multipart ETag, or a withheld replica ETag, returns `false` because no
|
||||
/// corruption can be concluded from it. An encrypted source's keyed
|
||||
/// fingerprint is also 32 hex characters and cannot be told apart from an MD5
|
||||
/// here, so callers must exempt encrypted sources before asking (the ecstore
|
||||
/// resyncer does so through its source-encryption classifier).
|
||||
pub fn single_part_replica_etag_mismatch(source_etag: Option<&str>, replica_etag: Option<&str>) -> bool {
|
||||
let Some(source) = source_etag.map(trim_etag) else {
|
||||
return false;
|
||||
|
||||
@@ -2740,6 +2740,7 @@ where
|
||||
Err(e) => {
|
||||
match classify_scanner_leader_lock_failure(&e) {
|
||||
ScannerLeaderLockFailure::Contended => {
|
||||
clear_scanner_transient_cycle_recovery_status();
|
||||
record_scanner_cycle_schedule_role("follower");
|
||||
record_scanner_leader_lock_state("contended");
|
||||
global_metrics().record_scanner_leader_liveness("contended", false, "").await;
|
||||
|
||||
@@ -353,6 +353,17 @@ fn publish_scanner_cleanup_failure(reason: String, expected: u64) {
|
||||
let _ = publish_scanner_cleanup_status(status, Some(expected));
|
||||
}
|
||||
|
||||
pub(super) fn clear_scanner_transient_cycle_recovery_status() {
|
||||
// Snapshot the version before reading status so any concurrent publish rejects the CAS below.
|
||||
let version = scanner_cleanup_status_version();
|
||||
let status = scanner_cycle_recovery_status();
|
||||
// A follower can discard its failed leadership observation. Durable recovery
|
||||
// markers are revalidated before any later leadership claim.
|
||||
if status.classification.is_none() && matches!(status.state.as_str(), "transient" | "paused") {
|
||||
let _ = publish_scanner_cleanup_status(recovery_status("healthy", None, false), Some(version));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_scanner_usage_floor_failure(reason: String) {
|
||||
let previous = scanner_cycle_recovery_status();
|
||||
let same_failure = previous.classification.as_deref() == Some(USAGE_FLOOR_LOAD_FAILED);
|
||||
|
||||
@@ -81,6 +81,159 @@ async fn persisted_state(store: &Arc<ECStore>) -> Vec<(Option<Vec<u8>>, DataUsag
|
||||
state
|
||||
}
|
||||
|
||||
async fn seed_transient_cycle_read_failure() -> ScannerCycleRecoveryStatus {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let error = EcstoreError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), DATA_USAGE_BLOOM_NAME_PATH.clone());
|
||||
let expected_error = format!("Scanner error: failed to inspect scanner cycle state: {error}");
|
||||
store
|
||||
.read_errors
|
||||
.lock()
|
||||
.await
|
||||
.insert(memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()), error);
|
||||
let ScannerCycleStateStartup::Transient(error) = load_scanner_cycle_state_for_startup(store).await else {
|
||||
panic!("a quorum outage must produce transient cycle recovery");
|
||||
};
|
||||
assert_eq!(error.to_string(), expected_error);
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.state, "transient");
|
||||
assert!(status.retryable);
|
||||
assert_eq!(status.classification, None);
|
||||
status
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_8297_contended_follower_clears_only_transient_cycle_recovery() {
|
||||
temp_env::async_with_vars([("RUSTFS_LOCK_ACQUIRE_TIMEOUT", Some("1"))], async {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader");
|
||||
let transient = seed_transient_cycle_read_failure().await;
|
||||
assert!(!record_scanner_cycle_recovery_retry(MAX_SCANNER_CYCLE_RECOVERY_RETRIES));
|
||||
let paused = scanner_cycle_recovery_status();
|
||||
assert_eq!(paused.state, "paused");
|
||||
let mut cases = vec![(transient, true), (paused, true)];
|
||||
for (state, classification, retryable) in [
|
||||
("recovery-required", None, false),
|
||||
("blocked", None, false),
|
||||
("blocked", Some("corrupt"), false),
|
||||
("cleanup-pending", None, true),
|
||||
("cleanup-pending", Some("corrupt"), true),
|
||||
("usage_floor_load_failed", Some("usage_floor_load_failed"), true),
|
||||
("usage_floor_recovery_pending", Some("legacy_empty_usage_floor"), true),
|
||||
("paused", Some("usage_floor_load_failed"), false),
|
||||
("cache_cycle_ahead", Some("cache_cycle_ahead"), true),
|
||||
] {
|
||||
cases.push((
|
||||
ScannerCycleRecoveryStatus {
|
||||
state: state.to_string(),
|
||||
classification: classification.map(str::to_string),
|
||||
retry_count: 5,
|
||||
max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
|
||||
retryable,
|
||||
reason: Some("preserve recovery state".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
));
|
||||
}
|
||||
for (status, clear) in cases {
|
||||
let expected = serde_json::to_value(&status).expect("expected recovery status");
|
||||
publish_scanner_cleanup_status(status, None).expect("seed local recovery status");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
run_data_scanner_with_maintenance_state(
|
||||
CancellationToken::new(),
|
||||
store.clone(),
|
||||
ScannerMaintenanceFeatures::default(),
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("contended scanner iteration must finish")
|
||||
.expect("lock contention is a normal follower outcome");
|
||||
assert_eq!(scanner_cycle_schedule_status().execution_role, "follower");
|
||||
let actual = scanner_cycle_recovery_status();
|
||||
if clear {
|
||||
assert_eq!(actual.state, "healthy", "stale follower recovery: {expected}");
|
||||
assert_eq!(actual.retry_count, 0);
|
||||
assert!(!actual.retryable);
|
||||
} else {
|
||||
assert_eq!(serde_json::to_value(actual).expect("actual recovery status"), expected);
|
||||
}
|
||||
assert!(!guard.is_lock_lost(), "follower must not revoke the leader");
|
||||
assert_eq!(persisted_state(&store).await, before, "follower must not rewrite durable recovery state");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn issue_8297_supervisor_recovers_paused_follower_without_restart() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_LOCK_ACQUIRE_TIMEOUT", Some("1")),
|
||||
(ENV_SCANNER_START_DELAY_SECS, Some("0")),
|
||||
(ENV_SCANNER_CYCLE, Some("1")),
|
||||
],
|
||||
async {
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
seed_cleanup(&store, "cleanup-pending").await;
|
||||
let before = persisted_state(&store).await;
|
||||
let lock = store
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
|
||||
.await
|
||||
.expect("leader lock");
|
||||
let guard = lock
|
||||
.get_write_lock_quiet(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold live leader");
|
||||
for attempt in 1..=MAX_SCANNER_CYCLE_RECOVERY_RETRIES {
|
||||
seed_transient_cycle_read_failure().await;
|
||||
assert_eq!(record_scanner_cycle_recovery_retry(attempt), attempt < MAX_SCANNER_CYCLE_RECOVERY_RETRIES);
|
||||
}
|
||||
assert_eq!(scanner_cycle_recovery_status().state, "paused");
|
||||
assert!(!crate::scanner_runtime_initialized(), "test must own the scanner runtime");
|
||||
let ctx = CancellationToken::new();
|
||||
let _cancel_on_drop = ctx.clone().drop_guard();
|
||||
init_data_scanner_with_storage(ctx.clone(), store.clone()).await;
|
||||
assert!(crate::scanner_runtime_initialized());
|
||||
let recovered = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while scanner_cycle_recovery_status().state != "healthy" {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
ctx.cancel();
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while crate::scanner_runtime_initialized() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled scanner supervisor must stop");
|
||||
recovered.expect("follower must recover without another five-minute pause or process restart");
|
||||
assert_eq!(scanner_cycle_schedule_status().execution_role, "follower");
|
||||
let status = scanner_cycle_recovery_status();
|
||||
assert_eq!(status.retry_count, 0);
|
||||
assert!(!status.retryable);
|
||||
assert!(!guard.is_lock_lost(), "supervisor must not revoke the leader");
|
||||
assert_eq!(persisted_state(&store).await, before, "supervisor must leave durable recovery untouched");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
}
|
||||
|
||||
async fn run_disabled_startup(ctx: CancellationToken, store: Arc<ECStore>) {
|
||||
let initialized_before = crate::scanner_runtime_initialized();
|
||||
let cleanup = init_scanner_with_recovery(ctx, store, false).await;
|
||||
|
||||
@@ -51,7 +51,7 @@ thiserror = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
|
||||
liblzma = { version = "0.4.8", features = ["static"] }
|
||||
liblzma = { version = "0.4.9", features = ["static"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-tokio-tar = { workspace = true }
|
||||
|
||||
@@ -8,6 +8,8 @@ collection:
|
||||
|
||||
## Operations
|
||||
|
||||
- [Table maintenance periodic queueing](operations/s3-tables-maintenance-scheduler.md) — opt-in scheduling, limits, worker handoff, and recovery.
|
||||
|
||||
For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity,
|
||||
see [PUT cleanup isolation](operations/put-cleanup-isolation.md).
|
||||
|
||||
|
||||
@@ -94,7 +94,9 @@ recoverable job and configuration records remain separate objects. The worker
|
||||
performs metadata and unreferenced-object cleanup without changing the catalog
|
||||
pointer. The direct `/maintenance/metadata` endpoint, including snapshot
|
||||
expiration and compaction plan/commit, remains object-backed only. A configured
|
||||
`background-enabled` table still requires explicit scheduler and worker ticks.
|
||||
`background-enabled` table still requires explicit worker ticks. The standalone
|
||||
server has opt-in, allowlisted periodic queue admission; see the
|
||||
[scheduler runbook](../operations/s3-tables-maintenance-scheduler.md).
|
||||
|
||||
Renamed tables retain their original metadata paths. Durable-strong scheduling
|
||||
and cleanup fail closed with a conflict if another table shares a scanned
|
||||
@@ -137,7 +139,7 @@ when a rename removes its old path.
|
||||
| Compaction planning | Preview / controlled | Partition-local and sort-order-local binpack candidates for Parquet; never mixes partitions or sort orders in one group. |
|
||||
| Delete-file or row-level compaction planning | Preview / controlled | Position or equality delete files force machine-readable planning into manual review. |
|
||||
| Compaction commit | Preview / controlled | Commits a safe partition-local Parquet rewrite preserving sort order IDs. |
|
||||
| Built-in periodic scheduler | Not claimed | Ticks are operator-triggered; no continuous in-process scheduling. |
|
||||
| Built-in periodic scheduler | Preview / queue-only | Default off; allowlisted, paged admission with jitter, per-pass quotas, current-job checks, and cooperative drain. Workers and failed/expired-job recovery remain operator-triggered; no automatic execution or hard per-table I/O deadline. |
|
||||
| Delete-file or row-level compaction execution | Not claimed | Manual-review maintenance item. |
|
||||
|
||||
## Recovery And Strong Backing Matrix
|
||||
|
||||
@@ -9,37 +9,42 @@ creating another with the same name does not transfer the old task to the new bu
|
||||
|
||||
## Incomplete bucket discovery
|
||||
|
||||
An administrator heal of the cluster or a selected pool/set must enumerate a complete bucket inventory before traversal. A bucket directory present on fewer disks than the namespace write quorum can be omitted from the ordinary bucket list even when every disk answers. Administrator discovery preserves that incomplete result instead of reporting an empty successful traversal. A pool/set task checks only its selected set; unavailable unrelated pools do not invalidate that scope.
|
||||
An administrator heal of the cluster or a selected pool/set enumerates bucket candidates from every selected disk, including directories below the ordinary namespace write quorum. Every selected disk must answer before discovery is complete. A pool/set task checks only its selected set; unavailable unrelated pools do not invalidate that scope. Candidate discovery grants no permission to recreate a bucket. Ordinary S3 and scanner listings retain their quorum rules.
|
||||
|
||||
Incomplete discovery uses the existing bounded task retry budget. Once exhausted, the canonical outcome is `execution.state=aborted`, `execution.reason=untraversable`, and `coverage=partial`; the legacy summary is `stopped`. Object counters can remain zero because traversal never started. A genuinely empty, completely enumerated namespace still finishes successfully. Dry runs use the same coverage checks.
|
||||
|
||||
A pool/set task persists its resolved bucket scope only after complete discovery. Resumed checkpoints keep their original frozen scope, including a valid empty scope. This change does not reinterpret historical checkpoints or terminal reports; start a fresh task to verify coverage after upgrading.
|
||||
Mutating cluster and pool/set tasks persist their resolved bucket scope only after complete discovery. Resumed checkpoints keep their original frozen scope, including a valid empty scope. A cluster task pins every bucket's incarnation before its first mutation and checkpoints its successful contiguous prefix. Checkpoints are batched into at most 32 intervals per traversal to bound inventory rewriting; an interrupted interval is replayed under the same generations. A fresh task is required to include buckets created after discovery. Cluster dry runs retain their observation-only behavior without requiring bucket incarnation metadata.
|
||||
|
||||
## Restoring a missing bucket directory
|
||||
|
||||
For a bucket that is still intended to exist but has lost its directory on one drive:
|
||||
|
||||
1. Check that the peer drives and authoritative bucket metadata remain available. Identify the affected bucket explicitly.
|
||||
2. Start an authenticated administrator heal for that bucket with `recursive=true` and `recreate=true`. A deep scan (`scanMode=2`) additionally checks data integrity. For the v3 API, the start endpoint is `POST /rustfs/admin/v3/heal/<bucket>`; retain the returned `clientToken` and poll the same endpoint with that query parameter.
|
||||
1. Check that the peer drives and authoritative bucket metadata remain available.
|
||||
2. Start an authenticated administrator heal with `recursive=true`. For the v3 API, use `POST /rustfs/admin/v3/heal/` for the cluster or `POST /rustfs/admin/v3/heal/<bucket>` for one bucket. A deep scan (`scanMode=2`) additionally checks data integrity. Retain the returned `clientToken` and poll the same endpoint with that query parameter.
|
||||
3. Inspect both execution and coverage in the terminal outcome. A readable object on a healthy peer alone does not establish that the missing copy was restored.
|
||||
4. Verify recovered object contents and lengths against known hashes. In an isolated recovery test, stop the original healthy source and read through the repaired target to demonstrate independent recovery. Do not take a production source offline merely to inspect a heal result.
|
||||
|
||||
Default bucket heal still enforces its existing namespace quorum checks. `recreate` is an explicit recovery option, not permission to revive a bucket deleted through S3. Bucket incarnation checks continue to reject stale tasks after deletion and same-name recreation. A cluster-wide `recreate` request also requires complete discovery; when discovery is incomplete, use the explicit affected-bucket path after resolving its authority.
|
||||
Default administrator bucket healing can reconstruct missing directories when every selected disk is reachable, each selected set retains a bucket directory, and persisted metadata proves the current generation was committed and has not been retired. The lifecycle fence remains held through reconstruction. Missing, corrupt, uncommitted, retired, or mismatched authority fails closed. Dry runs leave directories unchanged. `recreate=true` remains an explicit recovery option; it is not permission to revive a bucket deleted through S3. Bucket incarnation checks reject stale tasks after deletion and same-name recreation.
|
||||
|
||||
Legacy objects without independent storage commitments can be physically repaired while their canonical dispositions remain `unknown`. Do not infer a verified repair count from progress counters; see [heal terminal reports](../architecture/heal-terminal-reports.md).
|
||||
|
||||
## Recovery records and stale tokens
|
||||
|
||||
Pending records use schema 3 in `.rustfs.sys/root-heal-<task-id>.json` on the existing
|
||||
Pending records use schema 4 in `.rustfs.sys/root-heal-<task-id>.json` on the existing
|
||||
coordinator disk. A bucket record includes `bucket_incarnation_id`. Options, task ID,
|
||||
and the remaining retry budget retain their existing recovery semantics.
|
||||
and the remaining retry budget retain their existing recovery semantics. A cluster
|
||||
record additionally carries its frozen bucket generations and completed prefix.
|
||||
Cluster records with this scope have a 16 MiB limit; other intents and legacy
|
||||
terminal markers retain the 64 KiB limit. Initial cluster snapshots reserve 1 MiB
|
||||
for later progress and bounded object outcomes. Oversized inventories fail before repair.
|
||||
|
||||
A missing bucket, changed incarnation, or legacy bucket record without a usable
|
||||
identity becomes a durable `Failed` terminal result containing
|
||||
`stale_bucket_incarnation`. The original token remains queryable under the existing
|
||||
terminal retention policy. Recovery never fills a legacy task with the identity of
|
||||
the bucket currently using its name. Existing schema 1 cluster records and schema 2
|
||||
non-bucket records keep their recovery behavior. Unsupported, corrupt, and oversized
|
||||
the bucket currently using its name. Legacy schema 1–3 cluster intents lack a frozen
|
||||
generation inventory and require a fresh task after upgrade. Other supported legacy
|
||||
intents and terminal reports remain readable. Unsupported, corrupt, and oversized
|
||||
records remain intact and receive a sibling
|
||||
`quarantined-root-heal-{intent|terminal}-<task-id>.json` marker. The marker fences the
|
||||
original task ID while allowing an independent compensation heal with a fresh ID.
|
||||
@@ -65,15 +70,16 @@ missing or nil identities and unsigned bodies are rejected. Older nodes return
|
||||
`Unimplemented`; senders do not retry through an unfenced method. Upgrade all
|
||||
participating nodes before expecting these bucket heals to complete. Unscoped
|
||||
RPC bodies retain their prior canonical encoding. Older coordinators do not
|
||||
understand schema 3 pending records. Finish or cancel pending administrator tasks
|
||||
understand newer pending record schemas. Finish or cancel pending administrator tasks
|
||||
before downgrading a coordinator; do not rewrite the schema to force replay.
|
||||
|
||||
The lifecycle fence remains mandatory even when `nolock` disables the optional
|
||||
object namespace lock. Deployments with namespace locking disabled cannot perform
|
||||
incarnation-bound bucket healing.
|
||||
|
||||
This contract covers explicit administrator **bucket** tasks. Cluster, prefix,
|
||||
object, scanner, and replacement tasks retain their existing admission semantics.
|
||||
The generation-bound repair contract covers explicit administrator **bucket** tasks
|
||||
and the frozen buckets of mutating administrator cluster tasks. Prefix, object,
|
||||
scanner, and replacement tasks retain their existing admission semantics.
|
||||
It does not introduce a new object-generation protocol or claim rejection of every
|
||||
already-dispatched native syscall after distributed lock loss; see the existing
|
||||
[heal concurrency model](../architecture/heal-concurrency-model.md).
|
||||
|
||||
@@ -24,6 +24,17 @@ Clients that require an ETag for these older or multipart objects must issue an
|
||||
authorized HeadObject request, including the SSE-C key when required. GET,
|
||||
HEAD, and conditional requests continue to use the object's stored ETag.
|
||||
|
||||
## Replication
|
||||
|
||||
Bucket and site replication forward the stored ETag on `x-rustfs-source-etag`
|
||||
(with its `x-minio-source-*` twin). A target that adopts it (RustFS, MinIO)
|
||||
reports the fingerprint for the replica; a target that computes its own ETag
|
||||
(AWS S3 and most compatible stores) reports the MD5 of the plaintext it
|
||||
received, which never equals the fingerprint. The post-PUT replica ETag audit
|
||||
therefore skips encrypted sources, and the checksum header on the replication
|
||||
PUT remains their integrity check. See
|
||||
`docs/operations/replication-outbound-transport.md`.
|
||||
|
||||
## Existing data and migration
|
||||
|
||||
The upgrade does not rewrite existing object metadata or erase fingerprints
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
|
||||
**Use this when:** an object GET returns `ServiceUnavailable` near the namespace lock acquisition deadline while PUT or Heal is active on the same key.
|
||||
|
||||
## Client-visible acquisition timeouts
|
||||
|
||||
The canonical `ApiError` conversion in `rustfs/src/error.rs` retains HTTP 503 and the S3 `ServiceUnavailable` code for typed lock acquisition timeouts, and adds `Retry-After: 1`. The header is a minimum retry delay, not a receipt that the competing owner has released its lock. Use bounded exponential backoff with jitter and keep the client's overall request budget.
|
||||
|
||||
This advice does not change the lock acquisition budget or shorten a competing writer's commit tail. It is not added to unrelated disk timeouts, quorum failures, or errors that merely contain timeout text. A 503 near five seconds is insufficient to attribute the request to a namespace lock: disk metadata operations also have a five-second default budget. Correlate the typed lock-failure counters and acquisition events below before changing contention policy.
|
||||
|
||||
## Capture one acquisition and its competing owners
|
||||
|
||||
Enable the existing diagnostic switch before starting the process and add the diagnostic target to the deployment's log filter:
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
- For a single-part object, the checksum the source object was uploaded with, forwarded as its `x-amz-checksum-<algorithm>` header (the value the source verified on upload). A multipart replica is rebuilt through CreateMultipartUpload/UploadPart and carries no object-level checksum header. Managed-SSE objects forward none.
|
||||
- On a PUT that carries Object Lock parameters and no forwarded checksum: `Content-MD5` derived from the source ETag, or an SDK CRC32 checksum when the ETag is not the MD5 of the wire bytes (rustfs#7082).
|
||||
- The source ETag, mtime and version id on `x-rustfs-source-*` headers (with `x-minio-source-*` twins), and the Object Lock mode, retain-until date and legal hold of the source version when present.
|
||||
- After the PUT, the target's ETag is compared with the source ETag when both are plain single-part MD5s; a mismatch fails the replication instead of reporting a corrupted replica as COMPLETED.
|
||||
- After the PUT, the target's ETag is compared with the source ETag when both are plain single-part MD5s and the source is unencrypted; a mismatch fails the replication instead of reporting a corrupted replica as COMPLETED. A managed-SSE (SSE-S3 / SSE-KMS) or SSE-C source is not audited this way: its stored ETag is a keyed fingerprint (new writes, see `docs/operations/encrypted-etags.md`) or a legacy plaintext MD5, and the target re-encrypts what it receives, so the two ETags are never comparable. The checksum header on the replication PUT remains the integrity check for those objects.
|
||||
|
||||
## Target classes and their known requirements
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Table Maintenance Periodic Queueing
|
||||
|
||||
**Use this when:** enabling automatic maintenance job admission or recovering a stalled queue.
|
||||
|
||||
The server can periodically **queue** maintenance jobs for explicitly selected table buckets. It does not run workers, delete objects, expire snapshots, or compact data automatically. Workers remain an explicit operator action. Both object-backed and durable-strong catalogs use the existing maintenance configuration and job format.
|
||||
|
||||
## Enablement
|
||||
|
||||
The standalone server reads these variables at startup; the embedded server does not start this scheduler.
|
||||
|
||||
| Variable | Default | Accepted value |
|
||||
|---|---|---|
|
||||
| `RUSTFS_TABLE_MAINTENANCE_SCHEDULER_ENABLED` | `false` | `true` or `false`; invalid values fail startup |
|
||||
| `RUSTFS_TABLE_MAINTENANCE_SCHEDULER_BUCKETS` | None | Required when enabled: comma-separated, 1–32 distinct valid bucket names |
|
||||
| `RUSTFS_TABLE_MAINTENANCE_SCHEDULER_INTERVAL_SECONDS` | `300` | Integer from 30 to 86400 |
|
||||
|
||||
Example:
|
||||
|
||||
```sh
|
||||
export RUSTFS_TABLE_MAINTENANCE_SCHEDULER_ENABLED=true
|
||||
export RUSTFS_TABLE_MAINTENANCE_SCHEDULER_BUCKETS=analytics-tables
|
||||
export RUSTFS_TABLE_MAINTENANCE_SCHEDULER_INTERVAL_SECONDS=300
|
||||
```
|
||||
|
||||
The allowlist does not create or enable table buckets. Each bucket must already have an active catalog. The effective bucket default or table override must set `background-enabled=true` and `worker-paused=false`. Deletion remains disabled by default; queueing honors the existing `delete-enabled` setting but never executes the queued operation.
|
||||
|
||||
## Admission And Limits
|
||||
|
||||
- The first pass is delayed by the configured interval plus 0–10% jitter. Each completed pass waits again; missed ticks do not produce catch-up bursts.
|
||||
- Each process runs one owned pass at a time, visiting at most eight distinct tables and 64 discovery steps. Bucket turns rotate and namespace/table cursors persist between passes. Restart resumes discovery from the beginning; persisted jobs still prevent duplicate admission.
|
||||
- A 30-second admission deadline stops starting further store operations. An in-flight store call drains without cancellation or overlap. This is **not** a hard wall-clock, memory, or per-table object-I/O limit: existing planning and history inspection costs still depend on the table.
|
||||
- The scheduler only admits a first job or a new job after a successful previous job and the configured cooldown. It does not reset failed retry budgets, renew abandoned queues, recover expired workers, or clear quarantine.
|
||||
- Before publishing, the store rechecks table identity, enabled/unpaused configuration, the observed current job, and complete current/latest report publication under its existing locks. A concurrent manual run, rename, or identity change conflicts rather than overwriting that state.
|
||||
|
||||
Use the same interval on enabled nodes. The existing distributed catalog locks coordinate claims; this is not a cluster-wide throughput quota. A busy or corrupt table is reported as a degraded pass and does not remove other buckets from discovery.
|
||||
|
||||
## Inspection And Recovery
|
||||
|
||||
Inspect the existing endpoint:
|
||||
|
||||
```text
|
||||
GET /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/scheduler
|
||||
```
|
||||
|
||||
Use its current job, quarantine status, and recommended actions. An eligible queue can be consumed through the authenticated `POST .../maintenance/worker/run` endpoint. If a job failed, a queue/worker lease expired, or a previous manual invocation left a Disabled/Paused control report, resolve the underlying condition and explicitly invoke `POST .../maintenance/scheduler/run` or the recommended worker/quarantine operation. Automatic admission deliberately leaves these states unchanged.
|
||||
|
||||
A partial current/latest publication requires recovery before periodic admission; repeated ticks do not repair or discard it. Review retained reports and use the existing controlled maintenance/recovery APIs. Do not delete catalog sidecars directly.
|
||||
|
||||
To pause admission for a table, set `worker-paused=true` in its effective maintenance configuration. To stop the process-wide scheduler, disable the startup flag and restart. Shutdown stops new admissions and waits for owned queueing work; if completion cannot be confirmed within the normal shutdown grace, it retains unclean-shutdown markers and logs `drain_unconfirmed`. A deadline is not proof that a mutation was cancelled.
|
||||
|
||||
This scheduler does not claim automatic worker retry, cross-node worker takeover, bounded history retention, or a fully managed maintenance service.
|
||||
@@ -130,6 +130,7 @@ The request must be authenticated with an admin identity that has `ServerInfoAdm
|
||||
|---|---|
|
||||
| `runtime_config` | Effective runtime controls and their value sources. |
|
||||
| `cycle_schedule` | Current effective cycle interval and clean-idle backoff state. |
|
||||
| `cycle_recovery` | This node's cycle-state recovery observation, retry budget, and reason. |
|
||||
| `metrics` | Scanner work, pressure, checkpoint, lifecycle, replication, heal, bitrot, and alert counters. |
|
||||
| `data_movement_pause` | Global-pause policy, current movement reason, operation epoch, start time, duration, and estimated movement work items. |
|
||||
| `pause_backlog` | Replicated durable pause ledger, post-pause catch-up phase, rate window, retry state, thresholds, and active alert reasons. |
|
||||
@@ -190,6 +191,12 @@ Shared compacted accumulators defer periodic emission until the owning subtree c
|
||||
|
||||
A confirmed periodic save does not prove restart resumability. Adoption still validates bucket/set identity and, for non-empty caches, connected cache scope. Skipping completed subtrees additionally requires a validated coverage frontier; raw enumeration state alone does not prove completed subtree coverage.
|
||||
|
||||
### Cycle recovery on followers
|
||||
|
||||
Cycle recovery status is local to each node. A transient cycle-state read failure can exhaust the short retry budget and enter `paused`; the enabled scanner then retries leadership every five minutes or when recovery is explicitly woken. If the next lock attempt is contended or times out waiting for `leader.lock`, the node records a follower iteration and clears only its unclassified `transient` or `paused` observation. Normal follower retry cadence resumes without restarting the process.
|
||||
|
||||
This clears local retry bookkeeping, not persisted cycle state or recovery markers. Classified recovery states, cleanup-pending resets, and usage-floor failures remain unchanged. Any later leadership claim revalidates durable cycle state before scanning. A healthy follower recovery status does not prove that leader-dispatched scans are progressing; inspect the work counters separately.
|
||||
|
||||
## Usage State Reset
|
||||
|
||||
The supported break-glass route for rebuilding scanner usage state is `POST /v3/scanner/usage-state/reset` with body `{"mode":"full-rebuild"}`, authenticated as an admin identity holding `ConfigUpdateAdminAction` (route registered in `rustfs/src/admin/route_registration_test.rs`). Use it only after the scanner status shows a usage-floor load failure, a conflicting persisted usage floor, or an operator decision to discard the durable usage baseline and rebuild it from a full scanner pass.
|
||||
|
||||
Generated
+345
-288
File diff suppressed because it is too large
Load Diff
@@ -2561,24 +2561,21 @@ mod tests {
|
||||
for b in 1u8..8 {
|
||||
for c in 1u8..8 {
|
||||
let masks = [a, b, c];
|
||||
let snapshots = (0..3)
|
||||
.map(|source| rustfs_heal::ReplacementRecoverySnapshot {
|
||||
records: records
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(task, _)| masks[*task] & (1 << source) != 0)
|
||||
.map(|(_, record)| record.clone())
|
||||
.collect(),
|
||||
definitive: true,
|
||||
reason: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut snapshots = (0..3).map(|source| rustfs_heal::ReplacementRecoverySnapshot {
|
||||
records: records
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(task, _)| masks[*task] & (1 << source) != 0)
|
||||
.map(|(_, record)| record.clone())
|
||||
.collect(),
|
||||
definitive: true,
|
||||
reason: None,
|
||||
});
|
||||
let expected = (0..3).all(|left| {
|
||||
(left + 1..3).all(|right| {
|
||||
records[left].target_slots != records[right].target_slots || masks[left] & masks[right] != 0
|
||||
})
|
||||
});
|
||||
let mut snapshots = snapshots.into_iter();
|
||||
let local = snapshots.next().expect("three source snapshots");
|
||||
let cluster = aggregate_replacement_recovery_cluster_status(
|
||||
vec![local],
|
||||
|
||||
@@ -42,12 +42,9 @@ use s3s::{
|
||||
s3_error,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
future::Future,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::{future::Future, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::Instant;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
@@ -128,6 +125,13 @@ fn is_rebalance_start_fleet_proof_retryable(err: &StorageError) -> bool {
|
||||
&& err.to_string().contains(REBALANCE_START_FLEET_PROOF_MARKER)
|
||||
}
|
||||
|
||||
fn rebalance_init_failure_error(err: &StorageError) -> S3Error {
|
||||
if is_rebalance_start_fleet_proof_retryable(err) {
|
||||
return rebalance_fleet_proof_unavailable_error();
|
||||
}
|
||||
s3_error!(InternalError, "failed to initialize rebalance: {}", err)
|
||||
}
|
||||
|
||||
fn rebalance_start_failure_error(start_err: &StorageError, rollback_result: &Result<(), String>) -> S3Error {
|
||||
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(start_err) {
|
||||
if rollback_result.is_ok() {
|
||||
@@ -611,7 +615,20 @@ impl Operation for RebalanceStart {
|
||||
|
||||
let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
|
||||
|
||||
let id = match store.init_rebalance_start(buckets).await {
|
||||
let fleet_proof_deadline = Instant::now() + REBALANCE_START_FLEET_PROOF_RETRY_BUDGET;
|
||||
// A missing proof fails before init publishes metadata. Retrying the full
|
||||
// operation reacquires its gates and revalidates competing starts.
|
||||
let id = match retry_rebalance_start_operation(
|
||||
"init_rebalance_start",
|
||||
"",
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
fleet_proof_deadline,
|
||||
|| store.init_rebalance_start(buckets.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(StorageError::DecommissionAlreadyRunning) => {
|
||||
log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
|
||||
@@ -622,7 +639,7 @@ impl Operation for RebalanceStart {
|
||||
return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(s3_error!(InternalError, "failed to initialize rebalance: {}", e));
|
||||
return Err(rebalance_init_failure_error(&e));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -639,7 +656,6 @@ impl Operation for RebalanceStart {
|
||||
"admin rebalance state"
|
||||
);
|
||||
let notification_sys = current_notification_system();
|
||||
let fleet_proof_deadline = Instant::now() + REBALANCE_START_FLEET_PROOF_RETRY_BUDGET;
|
||||
for step in rebalance_start_steps(notification_sys.is_some()) {
|
||||
match step {
|
||||
RebalanceStartStep::PropagateFence => {
|
||||
@@ -1170,10 +1186,10 @@ mod rebalance_handler_tests {
|
||||
Body, HeaderMap, Method, Operation, Params, RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStart,
|
||||
RebalanceStartStep, RebalanceStatus, RebalanceStop, RebalanceStopPropagationStatus, S3ErrorCode, S3Request, StatusCode,
|
||||
Uri, build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status,
|
||||
is_rebalance_start_fleet_proof_retryable, rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets,
|
||||
rebalance_rollback_failure_message, rebalance_rollback_stop_failure_message, rebalance_start_failure_error,
|
||||
rebalance_start_rollback_error, rebalance_start_steps, rebalance_stop_target_id, rebalance_used_pct,
|
||||
retry_rebalance_start_operation, rollback_result_label, stop_rebalance_admission_first,
|
||||
is_rebalance_start_fleet_proof_retryable, rebalance_init_failure_error, rebalance_pool_used, rebalance_query_present,
|
||||
rebalance_remaining_buckets, rebalance_rollback_failure_message, rebalance_rollback_stop_failure_message,
|
||||
rebalance_start_failure_error, rebalance_start_rollback_error, rebalance_start_steps, rebalance_stop_target_id,
|
||||
rebalance_used_pct, retry_rebalance_start_operation, rollback_result_label, stop_rebalance_admission_first,
|
||||
};
|
||||
use crate::admin::storage_api::error::StorageError;
|
||||
use crate::admin::storage_api::rebalance::{
|
||||
@@ -1533,7 +1549,7 @@ mod rebalance_handler_tests {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_rebalance_start_retry_waits_for_fleet_proof() {
|
||||
async fn test_rebalance_init_retry_waits_for_fleet_proof() {
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -1544,12 +1560,12 @@ mod rebalance_handler_tests {
|
||||
let attempts = Arc::clone(&attempts);
|
||||
async move {
|
||||
retry_rebalance_start_operation(
|
||||
"load_rebalance_meta(start=true)",
|
||||
"rebalance-id",
|
||||
"init_rebalance_start",
|
||||
"",
|
||||
"request-id",
|
||||
"actor",
|
||||
"remote",
|
||||
std::time::Instant::now() + super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET,
|
||||
super::Instant::now() + super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET,
|
||||
move || {
|
||||
let attempts = Arc::clone(&attempts);
|
||||
async move {
|
||||
@@ -1557,7 +1573,7 @@ mod rebalance_handler_tests {
|
||||
if attempt == 0 {
|
||||
Err(StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER))
|
||||
} else {
|
||||
Ok(())
|
||||
Ok("initialized-once")
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1570,12 +1586,78 @@ mod rebalance_handler_tests {
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
|
||||
tokio::time::advance(super::REBALANCE_START_FLEET_PROOF_RETRY_DELAY).await;
|
||||
task.await
|
||||
let id = task
|
||||
.await
|
||||
.expect("retry task should not panic")
|
||||
.expect("fleet proof retry should eventually succeed");
|
||||
assert_eq!(id, "initialized-once");
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_rebalance_init_retry_exhaustion_returns_503_with_retry_after() {
|
||||
let mut attempts = 0;
|
||||
let started = super::Instant::now();
|
||||
let deadline = started + super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET;
|
||||
let err = retry_rebalance_start_operation::<(), _, _>(
|
||||
"init_rebalance_start",
|
||||
"",
|
||||
"request-id",
|
||||
"actor",
|
||||
"remote",
|
||||
deadline,
|
||||
|| {
|
||||
attempts += 1;
|
||||
std::future::ready(Err(StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER)))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a missing proof must exhaust the bounded retry budget");
|
||||
|
||||
assert_eq!(super::Instant::now() - started, super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET);
|
||||
assert_eq!(attempts, 7, "one immediate attempt plus six bounded readiness retries");
|
||||
let response = rebalance_init_failure_error(&err);
|
||||
assert_eq!(response.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.and_then(|headers| headers.get(http::header::RETRY_AFTER))
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(super::REBALANCE_START_RETRY_AFTER_SECS)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_rebalance_init_does_not_retry_other_failures() {
|
||||
for source in [
|
||||
StorageError::other("disk read failed"),
|
||||
StorageError::other("pool activation fleet capability proof expired before commit"),
|
||||
StorageError::RebalanceAlreadyRunning,
|
||||
] {
|
||||
let original = source.to_string();
|
||||
let mut source = Some(source);
|
||||
let started = super::Instant::now();
|
||||
let err = retry_rebalance_start_operation::<(), _, _>(
|
||||
"init_rebalance_start",
|
||||
"",
|
||||
"request-id",
|
||||
"actor",
|
||||
"remote",
|
||||
started + super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET,
|
||||
|| std::future::ready(Err(source.take().expect("non-readiness errors must not be retried"))),
|
||||
)
|
||||
.await
|
||||
.expect_err("non-readiness errors must be returned unchanged");
|
||||
assert_eq!(err.to_string(), original);
|
||||
assert_eq!(super::Instant::now(), started, "non-readiness failures must not wait");
|
||||
if !matches!(err, StorageError::RebalanceAlreadyRunning) {
|
||||
let response = rebalance_init_failure_error(&err);
|
||||
assert_eq!(response.code(), &S3ErrorCode::InternalError);
|
||||
assert!(response.message().is_some_and(|message| message.contains(&original)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_rollback_stop_failure_message_lists_failures() {
|
||||
let failures = vec![
|
||||
|
||||
@@ -23,6 +23,13 @@ const MAX_VERSIONS_EXCEEDED_CODE: &str = "MaxVersionsExceeded";
|
||||
const MAX_VERSIONS_EXCEEDED_MESSAGE: &str = "You've exceeded the limit on the number of versions you can create on this object";
|
||||
const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";
|
||||
const SLOW_DOWN_READ_MESSAGE: &str = "Resource requested is unreadable, please reduce your request rate";
|
||||
/// Public message for `StorageError::UnsupportedDisk` on a request path. The
|
||||
/// only request-time producer is the delete-transaction protocol when a
|
||||
/// participant answers Unimplemented (a peer still on a release without the
|
||||
/// recovery RPC, or a protocol-version mismatch); the variant's Display text
|
||||
/// ("drive does not support O_DIRECT") describes the startup drive probe and
|
||||
/// would send an operator mid-upgrade after the wrong cause.
|
||||
const UNSUPPORTED_PARTICIPANT_MESSAGE: &str = "A storage participant does not support this operation's protocol version. If the cluster is being upgraded, finish upgrading every node and retry.";
|
||||
|
||||
/// S3 error code for a request that names a KMS key the KMS does not hold.
|
||||
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
|
||||
@@ -599,7 +606,21 @@ fn error_chain_has_hyper_body_eof(err: &(dyn std::error::Error + 'static)) -> bo
|
||||
impl From<ApiError> for S3Error {
|
||||
fn from(err: ApiError) -> Self {
|
||||
let status = custom_error_status(&err.code);
|
||||
// A typed lock acquisition timeout is transient contention. Keep its
|
||||
// existing S3 code while advising a minimum delay before another attempt.
|
||||
let lock_timeout = err.code == S3ErrorCode::ServiceUnavailable
|
||||
&& err.source.as_deref().is_some_and(|source| {
|
||||
error_chain_find(source, |error| {
|
||||
matches!(error.downcast_ref::<LockError>(), Some(LockError::Timeout { .. })).then_some(())
|
||||
})
|
||||
.is_some()
|
||||
});
|
||||
let mut s3e = S3Error::with_message(err.code, err.message);
|
||||
if lock_timeout {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert(http::header::RETRY_AFTER, http::HeaderValue::from_static("1"));
|
||||
s3e.set_headers(headers);
|
||||
}
|
||||
if let Some(status) = status {
|
||||
s3e.set_status_code(status);
|
||||
}
|
||||
@@ -725,6 +746,9 @@ impl From<StorageError> for ApiError {
|
||||
StorageError::ObjectNameInvalid(_, _) => S3ErrorCode::InvalidArgument,
|
||||
StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyOwnedByYou,
|
||||
StorageError::StorageFull => S3ErrorCode::ServiceUnavailable,
|
||||
// Retryable on an upgraded node (or once the cluster upgrade
|
||||
// completes); see UNSUPPORTED_PARTICIPANT_MESSAGE.
|
||||
StorageError::UnsupportedDisk => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::SlowDown => S3ErrorCode::SlowDown,
|
||||
StorageError::FaultyDisk
|
||||
| StorageError::FaultyRemoteDisk
|
||||
@@ -768,6 +792,8 @@ impl From<StorageError> for ApiError {
|
||||
|
||||
let message = if matches!(&err, StorageError::QuotaExceeded { .. }) {
|
||||
err.to_string()
|
||||
} else if matches!(&err, StorageError::UnsupportedDisk) {
|
||||
UNSUPPORTED_PARTICIPANT_MESSAGE.to_string()
|
||||
} else if matches!(&err, StorageError::MaxVersionsExceeded)
|
||||
|| (code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)))
|
||||
{
|
||||
@@ -920,6 +946,108 @@ mod tests {
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
|
||||
#[tokio::test]
|
||||
async fn lock_timeout_response_advises_retry_without_exposing_resource() {
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
let storage_error =
|
||||
StorageError::Lock(LockError::timeout("private-bucket/private-key@latest", std::time::Duration::from_secs(5)));
|
||||
let error = S3Error::from(ApiError::from(storage_error));
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
let source = error.source().expect("retain the typed storage cause");
|
||||
assert!(source.downcast_ref::<StorageError>().is_some());
|
||||
assert!(error_chain_has_type::<LockError>(source));
|
||||
let response = error.to_http_response().expect("serialize lock timeout response");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
|
||||
let body = response
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("read timeout error XML")
|
||||
.to_bytes();
|
||||
let body = String::from_utf8(body.to_vec()).expect("S3 XML is UTF-8");
|
||||
assert!(body.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(!body.contains("private-bucket"));
|
||||
assert!(!body.contains("private-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_participant_is_retryable_with_an_actionable_message() {
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
let error = S3Error::from(ApiError::from(StorageError::UnsupportedDisk));
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.message(), Some(UNSUPPORTED_PARTICIPANT_MESSAGE));
|
||||
assert!(matches!(
|
||||
error.source().and_then(|source| source.downcast_ref::<StorageError>()),
|
||||
Some(StorageError::UnsupportedDisk)
|
||||
));
|
||||
|
||||
let response = error
|
||||
.to_http_response()
|
||||
.expect("unsupported participant error must serialize");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(
|
||||
!response.headers().contains_key(http::header::RETRY_AFTER),
|
||||
"retrying the same node does not help until it or its peers are upgraded"
|
||||
);
|
||||
let body = response
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("error XML must be readable")
|
||||
.to_bytes();
|
||||
let xml = std::str::from_utf8(&body).expect("error XML must be UTF-8");
|
||||
assert!(xml.contains("<Code>ServiceUnavailable</Code>"));
|
||||
assert!(
|
||||
!xml.contains("O_DIRECT"),
|
||||
"the drive-probe Display text must not leak as the public message"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lock_timeout_response_preserves_wrapped_source_and_requires_503() {
|
||||
for code in [S3ErrorCode::ServiceUnavailable, S3ErrorCode::RequestTimeout] {
|
||||
let should_advise_retry = code == S3ErrorCode::ServiceUnavailable;
|
||||
let error = S3Error::from(ApiError {
|
||||
message: ApiError::error_code_to_message(&code),
|
||||
code: code.clone(),
|
||||
source: Some(Box::new(IoError::other(LockError::timeout(
|
||||
"bucket/object@latest",
|
||||
std::time::Duration::from_secs(5),
|
||||
)))),
|
||||
});
|
||||
assert_eq!(error.code(), &code);
|
||||
let source = error.source().expect("preserve wrapped lock timeout");
|
||||
assert!(error_chain_has_type::<LockError>(source));
|
||||
let response = error.to_http_response().expect("serialize wrapped timeout");
|
||||
assert_eq!(response.headers().contains_key(http::header::RETRY_AFTER), should_advise_retry);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_unavailable_responses_do_not_inherit_lock_retry_advice() {
|
||||
for failure in [
|
||||
StorageError::Timeout,
|
||||
StorageError::DiskNotFound,
|
||||
StorageError::FileCorrupt,
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: "bucket".to_owned(),
|
||||
object: "object".to_owned(),
|
||||
required: 2,
|
||||
achieved: 1,
|
||||
},
|
||||
StorageError::Io(IoError::other("Lock acquisition timeout for bucket/object after 5s")),
|
||||
StorageError::SlowDown,
|
||||
] {
|
||||
let error = S3Error::from(ApiError::from(failure));
|
||||
let response = error.to_http_response().expect("serialize unrelated storage failure");
|
||||
assert!(!response.headers().contains_key(http::header::RETRY_AFTER));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lock_503_mapping_records_operation_and_resource_class_without_resource_name() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -1771,6 +1899,7 @@ mod tests {
|
||||
),
|
||||
(StorageError::BucketExists("test".into()), S3ErrorCode::BucketAlreadyOwnedByYou),
|
||||
(StorageError::StorageFull, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::UnsupportedDisk, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::SlowDown, S3ErrorCode::SlowDown),
|
||||
(StorageError::FaultyDisk, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::FaultyRemoteDisk, S3ErrorCode::ServiceUnavailable),
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::{
|
||||
startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap},
|
||||
startup_runtime_sources,
|
||||
startup_services::StartupServiceRuntime,
|
||||
startup_shutdown::run_startup_shutdown_sequence,
|
||||
startup_shutdown::{StartupBackgroundTasks, run_startup_shutdown_sequence},
|
||||
};
|
||||
use rustfs_common::GlobalReadiness;
|
||||
use rustfs_object_capacity::capacity_manager::CapacityBackgroundTasks;
|
||||
@@ -130,6 +130,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
|
||||
readiness,
|
||||
} = lifecycle;
|
||||
let StartupServiceRuntime {
|
||||
table_maintenance,
|
||||
oidc_recovery,
|
||||
optional_runtimes,
|
||||
heartbeat,
|
||||
@@ -163,7 +164,10 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
|
||||
s3_shutdown_tx,
|
||||
console_shutdown_tx,
|
||||
optional_runtimes,
|
||||
capacity_tasks,
|
||||
StartupBackgroundTasks {
|
||||
capacity: capacity_tasks,
|
||||
table_maintenance,
|
||||
},
|
||||
shutdown_token,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -43,6 +43,7 @@ const LOG_COMPONENT_CONNECT: &str = "connect";
|
||||
const LOG_SUBSYSTEM_INVENTORY: &str = "inventory";
|
||||
|
||||
pub(crate) struct StartupServiceRuntime {
|
||||
pub(crate) table_maintenance: Option<crate::table_catalog::PeriodicMaintenanceRuntime>,
|
||||
pub(crate) oidc_recovery: Option<AbortOnDropHandle<()>>,
|
||||
pub(crate) optional_runtimes: OptionalRuntimeServices,
|
||||
pub(crate) heartbeat: Option<HeartbeatRuntime>,
|
||||
@@ -83,6 +84,7 @@ pub(crate) async fn init_startup_runtime_services(
|
||||
state_manager: Arc<ServiceStateManager>,
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
) -> Result<StartupServiceRuntime> {
|
||||
let maintenance_config = crate::table_catalog::PeriodicMaintenanceConfig::from_env().map_err(std::io::Error::other)?;
|
||||
init_kms_system(config, store.clone()).await?;
|
||||
|
||||
let optional_runtimes = init_optional_runtime_services(server_ctx.clone()).await?;
|
||||
@@ -102,7 +104,7 @@ pub(crate) async fn init_startup_runtime_services(
|
||||
init_deadlock_detector_runtime();
|
||||
|
||||
let buckets = init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
|
||||
let iam_bootstrap = init_iam_runtime(store.clone(), ctx.clone(), readiness, state_manager, server_ctx).await?;
|
||||
let iam_bootstrap = init_iam_runtime(store.clone(), ctx.clone(), readiness, state_manager, server_ctx.clone()).await?;
|
||||
|
||||
// Audit initialization requires the AppContext (server config + object store)
|
||||
// which is published by ensure_startup_after_iam inside init_iam_runtime.
|
||||
@@ -119,7 +121,11 @@ pub(crate) async fn init_startup_runtime_services(
|
||||
let heartbeat = start_heartbeat_runtime(heartbeat_config.clone(), heartbeat_nodes, &ctx)?;
|
||||
let inventory = start_inventory_runtime(heartbeat_config, heartbeat_nodes, inventory_drives, store, &ctx)?;
|
||||
|
||||
let table_maintenance =
|
||||
maintenance_config.map(|config| crate::table_catalog::PeriodicMaintenanceRuntime::spawn(config, server_ctx, &ctx));
|
||||
|
||||
Ok(StartupServiceRuntime {
|
||||
table_maintenance,
|
||||
oidc_recovery,
|
||||
optional_runtimes,
|
||||
heartbeat,
|
||||
|
||||
@@ -206,10 +206,10 @@ fn background_shutdown_steps(enable_scanner: bool, enable_heal: bool) -> Vec<Bac
|
||||
/// those to reach the end of their on-disk steps. Then record the shutdown as
|
||||
/// clean, so the next startup skips the unclean-restart erasure-set heal, but
|
||||
/// only if nothing was left unfinished: a detached mutation still running at
|
||||
/// the deadline can be cut short with the runtime, and the heal handoff may
|
||||
/// not have completed. Returns whether the markers were cleared.
|
||||
/// the deadline can be cut short with the runtime, and a background handoff
|
||||
/// may not have completed. Returns whether the markers were cleared.
|
||||
async fn finish_detached_mutations_and_clear_unclean_shutdown_markers<W, WaitFut, ClearFut>(
|
||||
heal_handoff_complete: bool,
|
||||
background_handoff_complete: bool,
|
||||
detached_mutation_timeout: std::time::Duration,
|
||||
wait_for_detached_mutations: W,
|
||||
clear_unclean_shutdown_markers: ClearFut,
|
||||
@@ -233,22 +233,31 @@ where
|
||||
"Detached mutations did not finish before shutdown; retaining unclean-shutdown markers"
|
||||
);
|
||||
}
|
||||
if !(heal_handoff_complete && detached_mutations_drained) {
|
||||
if !(background_handoff_complete && detached_mutations_drained) {
|
||||
return false;
|
||||
}
|
||||
clear_unclean_shutdown_markers.await;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) struct StartupBackgroundTasks {
|
||||
pub(crate) capacity: Option<CapacityBackgroundTasks>,
|
||||
pub(crate) table_maintenance: Option<crate::table_catalog::PeriodicMaintenanceRuntime>,
|
||||
}
|
||||
|
||||
pub(crate) async fn run_startup_shutdown_sequence(
|
||||
state_manager: &ServiceStateManager,
|
||||
shutdown_signal: ShutdownSignal,
|
||||
s3_shutdown_handle: Option<ShutdownHandle>,
|
||||
console_shutdown_handle: Option<ShutdownHandle>,
|
||||
optional_runtimes: OptionalRuntimeServices,
|
||||
capacity_tasks: Option<CapacityBackgroundTasks>,
|
||||
background_tasks: StartupBackgroundTasks,
|
||||
ctx: CancellationToken,
|
||||
) {
|
||||
let StartupBackgroundTasks {
|
||||
capacity: capacity_tasks,
|
||||
table_maintenance,
|
||||
} = background_tasks;
|
||||
ctx.cancel();
|
||||
|
||||
// Stop long-lived peer/disk background monitors while the runtime and tracing
|
||||
@@ -463,10 +472,14 @@ pub(crate) async fn run_startup_shutdown_sequence(
|
||||
console_shutdown_handle.shutdown().await;
|
||||
}
|
||||
shutdown_optional_runtime_services(optional_runtime_shutdowns).await;
|
||||
let maintenance_drained = match table_maintenance {
|
||||
Some(runtime) => runtime.shutdown(SHUTDOWN_TIMEOUT).await,
|
||||
None => true,
|
||||
};
|
||||
// FTP, SFTP and WebDAV can start deletes too: wait for detached mutations
|
||||
// only after every front end has stopped.
|
||||
finish_detached_mutations_and_clear_unclean_shutdown_markers(
|
||||
heal_handoff_complete,
|
||||
heal_handoff_complete && maintenance_drained,
|
||||
DETACHED_MUTATION_SHUTDOWN_TIMEOUT,
|
||||
wait_for_detached_mutations,
|
||||
rustfs_heal::heal::clear_unclean_shutdown_markers(),
|
||||
|
||||
@@ -460,6 +460,7 @@ pub(crate) mod workload {
|
||||
}
|
||||
|
||||
pub(crate) mod table {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::utils::check_valid_bucket_name_strict;
|
||||
pub(crate) use crate::storage::storage_api::ecstore_object::{WriteCommitGuard, WriteCompletion};
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod http {
|
||||
|
||||
@@ -13,11 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
mod config;
|
||||
mod periodic;
|
||||
mod periodic_runtime;
|
||||
mod planner;
|
||||
mod recovery;
|
||||
mod worker;
|
||||
|
||||
pub(crate) use config::*;
|
||||
pub(crate) use periodic_runtime::{PeriodicMaintenanceConfig, PeriodicMaintenanceRuntime};
|
||||
pub(crate) use planner::*;
|
||||
pub(crate) use recovery::*;
|
||||
pub(crate) use worker::*;
|
||||
|
||||
@@ -0,0 +1,812 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::super::*;
|
||||
use std::collections::BTreeSet;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::time::Duration as StdDuration;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const PERIODIC_MAINTENANCE_TABLES_PER_PASS: usize = 8;
|
||||
const PERIODIC_MAINTENANCE_DISCOVERY_STEPS_PER_PASS: usize = 64;
|
||||
|
||||
struct BucketCursor {
|
||||
bucket: String,
|
||||
namespace_cursor: Option<String>,
|
||||
namespace: Option<String>,
|
||||
table_cursor: Option<String>,
|
||||
}
|
||||
|
||||
impl BucketCursor {
|
||||
fn reset(&mut self) {
|
||||
self.namespace_cursor = None;
|
||||
self.namespace = None;
|
||||
self.table_cursor = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue-only admission. The caller owns this future until it returns: a
|
||||
/// deadline stops new operations, never cancels a catalog mutation in flight.
|
||||
pub(crate) struct PeriodicMaintenanceScheduler<B: TableCatalogObjectBackend> {
|
||||
store: ConfiguredTableCatalogStore<B>,
|
||||
buckets: Vec<BucketCursor>,
|
||||
next_bucket: usize,
|
||||
interval: StdDuration,
|
||||
scheduler_id: String,
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> PeriodicMaintenanceScheduler<B> {
|
||||
/// Bucket names and the interval have already passed runtime configuration validation.
|
||||
pub(crate) fn new(store: ConfiguredTableCatalogStore<B>, buckets: Vec<String>, interval: StdDuration) -> Self {
|
||||
Self {
|
||||
store,
|
||||
buckets: buckets
|
||||
.into_iter()
|
||||
.map(|bucket| BucketCursor {
|
||||
bucket,
|
||||
namespace_cursor: None,
|
||||
namespace: None,
|
||||
table_cursor: None,
|
||||
})
|
||||
.collect(),
|
||||
next_bucket: 0,
|
||||
interval,
|
||||
scheduler_id: format!("periodic-{}", uuid::Uuid::new_v4()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn run_pass(
|
||||
&mut self,
|
||||
cancellation: &CancellationToken,
|
||||
admission_deadline: Instant,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if self.buckets.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut visited = BTreeSet::new();
|
||||
let mut unavailable_buckets = BTreeSet::new();
|
||||
let mut first_error = None;
|
||||
for _ in 0..PERIODIC_MAINTENANCE_DISCOVERY_STEPS_PER_PASS {
|
||||
if !admission_open(cancellation, admission_deadline) || visited.len() == PERIODIC_MAINTENANCE_TABLES_PER_PASS {
|
||||
break;
|
||||
}
|
||||
let bucket_index = self.next_bucket;
|
||||
self.next_bucket = (bucket_index + 1) % self.buckets.len();
|
||||
if unavailable_buckets.contains(&bucket_index) {
|
||||
continue;
|
||||
}
|
||||
let table = match self.discover_next_table(bucket_index, cancellation, admission_deadline).await {
|
||||
Ok(Some(table)) => table,
|
||||
Ok(None) => continue,
|
||||
Err(error) => {
|
||||
if matches!(&error, TableCatalogStoreError::NotFound(_)) {
|
||||
// A namespace removed between pages must not pin this bucket's cursor forever.
|
||||
self.buckets[bucket_index].reset();
|
||||
}
|
||||
unavailable_buckets.insert(bucket_index);
|
||||
first_error.get_or_insert(error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !visited.insert((table.table_bucket.clone(), table.table_id.clone())) {
|
||||
continue;
|
||||
}
|
||||
if let Err(error) = self.queue_if_eligible(&table, cancellation, admission_deadline).await {
|
||||
// A removed/renamed table or one unhealthy bucket must not starve other buckets.
|
||||
first_error.get_or_insert(error);
|
||||
}
|
||||
}
|
||||
match first_error {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_next_table(
|
||||
&mut self,
|
||||
bucket_index: usize,
|
||||
cancellation: &CancellationToken,
|
||||
admission_deadline: Instant,
|
||||
) -> TableCatalogStoreResult<Option<TableEntry>> {
|
||||
let cursor = &mut self.buckets[bucket_index];
|
||||
if !admission_open(cancellation, admission_deadline) {
|
||||
return Ok(None);
|
||||
}
|
||||
if self
|
||||
.store
|
||||
.get_table_bucket(&cursor.bucket)
|
||||
.await?
|
||||
.is_none_or(|bucket| bucket.state != TableCatalogEntryState::Active)
|
||||
{
|
||||
cursor.reset();
|
||||
return Ok(None);
|
||||
}
|
||||
if cursor.namespace.is_none() {
|
||||
if !admission_open(cancellation, admission_deadline) {
|
||||
return Ok(None);
|
||||
}
|
||||
let page = self
|
||||
.store
|
||||
.list_namespaces_page(&cursor.bucket, cursor.namespace_cursor.as_deref(), NonZeroUsize::MIN)
|
||||
.await?;
|
||||
cursor.namespace_cursor = page.next_cursor;
|
||||
cursor.namespace = page.entries.into_iter().next().map(|namespace| namespace.namespace);
|
||||
cursor.table_cursor = None;
|
||||
}
|
||||
let Some(namespace) = cursor.namespace.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !admission_open(cancellation, admission_deadline) {
|
||||
return Ok(None);
|
||||
}
|
||||
let page = self
|
||||
.store
|
||||
.list_tables_page(&cursor.bucket, namespace, cursor.table_cursor.as_deref(), NonZeroUsize::MIN)
|
||||
.await?;
|
||||
cursor.table_cursor = page.next_cursor;
|
||||
if cursor.table_cursor.is_none() {
|
||||
cursor.namespace = None;
|
||||
}
|
||||
// Object-backed discovery can return an empty page with a continuation.
|
||||
Ok(page.entries.into_iter().next())
|
||||
}
|
||||
|
||||
async fn queue_if_eligible(
|
||||
&self,
|
||||
table: &TableEntry,
|
||||
cancellation: &CancellationToken,
|
||||
admission_deadline: Instant,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if !admission_open(cancellation, admission_deadline) {
|
||||
return Ok(());
|
||||
}
|
||||
let scheduler = self
|
||||
.store
|
||||
.get_table_maintenance_scheduler_report(&table.table_bucket, &table.namespace, &table.table)
|
||||
.await?;
|
||||
if scheduler.table_id != table.table_id {
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table identity changed during periodic maintenance discovery".to_string(),
|
||||
));
|
||||
}
|
||||
if !scheduler.background_enabled || scheduler.worker_paused || scheduler.quarantine.active {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(current) = scheduler.current_job.as_ref() {
|
||||
if current.status != TableMetadataMaintenanceJobStatus::Successful {
|
||||
// Periodic admission never renews an abandoned queue or resets a failed job's retry budget.
|
||||
return Ok(());
|
||||
}
|
||||
let finished_at = current
|
||||
.finished_at
|
||||
.as_deref()
|
||||
.and_then(|timestamp| OffsetDateTime::parse(timestamp, &time::format_description::well_known::Rfc3339).ok())
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid(
|
||||
"periodic maintenance requires a valid successful job completion timestamp".to_string(),
|
||||
)
|
||||
})?;
|
||||
let interval = time::Duration::try_from(self.interval)
|
||||
.map_err(|_| TableCatalogStoreError::Invalid("periodic maintenance interval is out of range".to_string()))?;
|
||||
let next_admission = finished_at.checked_add(interval).ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid("periodic maintenance completion timestamp is out of range".to_string())
|
||||
})?;
|
||||
if OffsetDateTime::now_utc() < next_admission {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
if !admission_open(cancellation, admission_deadline) {
|
||||
return Ok(());
|
||||
}
|
||||
self.store
|
||||
.queue_periodic_table_maintenance(table, self.scheduler_id.clone(), scheduler.current_job.as_ref())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn admission_open(cancellation: &CancellationToken, deadline: Instant) -> bool {
|
||||
!cancellation.is_cancelled() && Instant::now() < deadline
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::TestCatalogObjectBackend;
|
||||
use crate::table_catalog::tests::{test_bucket_entry, test_namespace_entry, test_table_entry};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
type TestStore = ConfiguredTableCatalogStore<TestCatalogObjectBackend>;
|
||||
|
||||
fn backings() -> [TableCatalogBackingMode; 2] {
|
||||
[TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong]
|
||||
}
|
||||
|
||||
fn enabled_config() -> TableMaintenanceConfig {
|
||||
TableMaintenanceConfig {
|
||||
background_enabled: true,
|
||||
delete_enabled: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_table(
|
||||
store: &TestStore,
|
||||
backend: &TestCatalogObjectBackend,
|
||||
bucket: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
config: Option<TableMaintenanceConfig>,
|
||||
) -> TableEntry {
|
||||
let namespace = Namespace::parse(namespace).expect("fixture namespace");
|
||||
let table = IdentifierSegment::parse(table).expect("fixture table");
|
||||
if store.get_table_bucket(bucket).await.expect("read fixture bucket").is_none() {
|
||||
store
|
||||
.put_table_bucket(test_bucket_entry(bucket))
|
||||
.await
|
||||
.expect("create fixture bucket");
|
||||
}
|
||||
if store
|
||||
.get_namespace(bucket, &namespace.public_name())
|
||||
.await
|
||||
.expect("read fixture namespace")
|
||||
.is_none()
|
||||
{
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.expect("create fixture namespace");
|
||||
}
|
||||
let metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
|
||||
let mut entry = test_table_entry(bucket, &namespace, &table, metadata.clone());
|
||||
entry.table_id = uuid::Uuid::new_v4().to_string();
|
||||
entry.table_uuid = entry.table_id.clone();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/{}", entry.table_id);
|
||||
store.create_table(entry.clone()).await.expect("create fixture table");
|
||||
backend
|
||||
.seed_object(bucket, &metadata, br#"{"metadata-log":[]}"#.to_vec())
|
||||
.await;
|
||||
if let Some(config) = config {
|
||||
store
|
||||
.put_table_maintenance_config(bucket, &namespace.public_name(), table.as_str(), config)
|
||||
.await
|
||||
.expect("configure fixture table");
|
||||
}
|
||||
entry
|
||||
}
|
||||
|
||||
async fn current(store: &TestStore, table: &TableEntry) -> Option<TableMetadataMaintenanceReport> {
|
||||
store
|
||||
.get_table_metadata_maintenance_report(
|
||||
&table.table_bucket,
|
||||
&table.namespace,
|
||||
&table.table,
|
||||
MAINTENANCE_JOB_ALIAS_CURRENT,
|
||||
)
|
||||
.await
|
||||
.expect("read current fixture job")
|
||||
}
|
||||
|
||||
async fn seed_report(backend: &TestCatalogObjectBackend, table: &TableEntry, report: &TableMetadataMaintenanceReport) {
|
||||
let namespace = Namespace::parse(&table.namespace).expect("fixture namespace");
|
||||
let name = IdentifierSegment::parse(&table.table).expect("fixture table");
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
for path in [
|
||||
paths.table_maintenance_job_path(&table.table_bucket, &namespace, &name, &table.table_id, &report.job.job_id),
|
||||
paths.table_maintenance_latest_job_path(&table.table_bucket, &namespace, &name, &table.table_id),
|
||||
paths.table_maintenance_current_job_path(&table.table_bucket, &namespace, &name, &table.table_id),
|
||||
] {
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(report).expect("serialize fixture report"))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn object_versions(backend: &TestCatalogObjectBackend) -> BTreeMap<(String, String), (Vec<u8>, String)> {
|
||||
backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.objects
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), (value.data.clone(), value.etag.clone())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn run_pass(scheduler: &mut PeriodicMaintenanceScheduler<TestCatalogObjectBackend>) -> TableCatalogStoreResult<()> {
|
||||
scheduler
|
||||
.run_pass(&CancellationToken::new(), Instant::now() + StdDuration::from_secs(60))
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_is_idempotent_and_never_runs_delete_worker() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let namespace = Namespace::parse(&table.namespace).expect("fixture namespace");
|
||||
let name = IdentifierSegment::parse(&table.table).expect("fixture table");
|
||||
let orphan = default_table_metadata_file_path(&namespace, &name, "00000.metadata.json");
|
||||
backend
|
||||
.seed_object(&table.table_bucket, &orphan, br#"{"metadata-log":[]}"#.to_vec())
|
||||
.await;
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store.clone(), vec![table.table_bucket.clone()], StdDuration::from_secs(300));
|
||||
|
||||
run_pass(&mut scheduler).await.expect("first periodic pass queues work");
|
||||
let queued = current(&store, &table).await.expect("queued job");
|
||||
assert_eq!(queued.job.status, TableMetadataMaintenanceJobStatus::Queued);
|
||||
assert_eq!(queued.job.operation, TableMetadataMaintenanceOperation::Delete);
|
||||
assert!(queued.job.worker_id.is_none());
|
||||
assert_eq!(queued.job.attempt, 0);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(&table.table_bucket, &orphan)
|
||||
.await
|
||||
.expect("read orphan")
|
||||
.is_some()
|
||||
);
|
||||
assert!(backend.state.lock().await.delete_attempts.is_empty());
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(&table.table_bucket, &table.namespace, &table.table)
|
||||
.await
|
||||
.expect("read unchanged table"),
|
||||
Some(table.clone())
|
||||
);
|
||||
|
||||
let versions = object_versions(&backend).await;
|
||||
run_pass(&mut scheduler).await.expect("repeated pass skips queued work");
|
||||
assert_eq!(
|
||||
object_versions(&backend).await,
|
||||
versions,
|
||||
"queue-only repetition must not rewrite history or aliases"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_respects_disabled_paused_and_inherited_configuration() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let disabled = seed_table(&store, &backend, "disabled", "sales", "orders", None).await;
|
||||
let paused = seed_table(
|
||||
&store,
|
||||
&backend,
|
||||
"paused",
|
||||
"sales",
|
||||
"orders",
|
||||
Some(TableMaintenanceConfig {
|
||||
worker_paused: true,
|
||||
..enabled_config()
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let inherited = seed_table(&store, &backend, "inherited", "sales", "orders", None).await;
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&TableCatalogObjectPaths::default().table_bucket_maintenance_config_path("inherited"),
|
||||
serde_json::to_vec(&enabled_config()).expect("serialize bucket defaults"),
|
||||
)
|
||||
.await;
|
||||
let mut scheduler = PeriodicMaintenanceScheduler::new(
|
||||
store.clone(),
|
||||
vec!["disabled".to_string(), "paused".to_string(), "inherited".to_string()],
|
||||
StdDuration::from_secs(300),
|
||||
);
|
||||
run_pass(&mut scheduler).await.expect("configuration-aware pass");
|
||||
assert!(
|
||||
current(&store, &disabled).await.is_none(),
|
||||
"disabled tables must not gain control history"
|
||||
);
|
||||
assert!(current(&store, &paused).await.is_none(), "paused tables must not gain control history");
|
||||
let queued = current(&store, &inherited)
|
||||
.await
|
||||
.expect("bucket defaults must enable admission");
|
||||
assert_eq!(queued.job.config_source, TableMaintenanceConfigSource::TableBucketDefault);
|
||||
assert_eq!(queued.job.status, TableMetadataMaintenanceJobStatus::Queued);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_never_recovers_failed_expired_or_quarantined_jobs() {
|
||||
for backing in backings() {
|
||||
for status in [
|
||||
TableMetadataMaintenanceJobStatus::Failed,
|
||||
TableMetadataMaintenanceJobStatus::Queued,
|
||||
TableMetadataMaintenanceJobStatus::Running,
|
||||
TableMetadataMaintenanceJobStatus::Successful,
|
||||
] {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let mut report = store
|
||||
.run_table_maintenance_scheduler_once("analytics", "sales", "orders", "fixture".to_string())
|
||||
.await
|
||||
.expect("seed real queued report")
|
||||
.report;
|
||||
report.job.status = status.clone();
|
||||
report.job.attempt = 4;
|
||||
report.job.max_retry_attempts = 3;
|
||||
report.job.scheduled_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH));
|
||||
report.job.heartbeat_at = report.job.scheduled_at.clone();
|
||||
report.job.finished_at = report.job.scheduled_at.clone();
|
||||
report.job.next_retry_after = None;
|
||||
if status == TableMetadataMaintenanceJobStatus::Successful {
|
||||
report.job.quarantine_enabled = true;
|
||||
report.job.quarantined_object_count = 1;
|
||||
report.job.quarantine_retention_seconds = 3600;
|
||||
}
|
||||
seed_report(&backend, &table, &report).await;
|
||||
let versions = object_versions(&backend).await;
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store.clone(), vec!["analytics".to_string()], StdDuration::from_secs(300));
|
||||
run_pass(&mut scheduler)
|
||||
.await
|
||||
.expect("manual recovery boundary is a normal skip");
|
||||
assert_eq!(
|
||||
object_versions(&backend).await,
|
||||
versions,
|
||||
"periodic passes must preserve failed budgets, expired leases and quarantine"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_success_cooldown_requires_valid_persisted_timestamp() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let mut report = store
|
||||
.run_table_maintenance_scheduler_once("analytics", "sales", "orders", "fixture".to_string())
|
||||
.await
|
||||
.expect("seed real report")
|
||||
.report;
|
||||
report.job.status = TableMetadataMaintenanceJobStatus::Successful;
|
||||
report.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::now_utc()));
|
||||
seed_report(&backend, &table, &report).await;
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store.clone(), vec!["analytics".to_string()], StdDuration::from_secs(300));
|
||||
let versions = object_versions(&backend).await;
|
||||
run_pass(&mut scheduler).await.expect("recent success is cooling down");
|
||||
assert_eq!(object_versions(&backend).await, versions);
|
||||
for timestamp in [None, Some("not-a-timestamp".to_string())] {
|
||||
report.job.finished_at = timestamp;
|
||||
seed_report(&backend, &table, &report).await;
|
||||
let versions = object_versions(&backend).await;
|
||||
assert!(matches!(run_pass(&mut scheduler).await, Err(TableCatalogStoreError::Invalid(_))));
|
||||
assert_eq!(object_versions(&backend).await, versions, "invalid cooldown must fail without a new job");
|
||||
}
|
||||
report.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH));
|
||||
seed_report(&backend, &table, &report).await;
|
||||
run_pass(&mut scheduler).await.expect("expired cooldown permits a new job");
|
||||
let queued = current(&store, &table).await.expect("next queue");
|
||||
assert_eq!(queued.job.status, TableMetadataMaintenanceJobStatus::Queued);
|
||||
assert_ne!(queued.job.job_id, report.job.job_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_pages_and_round_robins_buckets_across_passes() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let buckets = ["first", "second", "third"];
|
||||
let mut tables = Vec::new();
|
||||
for bucket in buckets {
|
||||
for index in 0..5 {
|
||||
tables.push(
|
||||
seed_table(
|
||||
&store,
|
||||
&backend,
|
||||
bucket,
|
||||
if index < 2 { "sales" } else { "support" },
|
||||
&format!("table{index}"),
|
||||
Some(enabled_config()),
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut scheduler = PeriodicMaintenanceScheduler::new(
|
||||
store.clone(),
|
||||
buckets.iter().map(|bucket| (*bucket).to_string()).collect(),
|
||||
StdDuration::from_secs(300),
|
||||
);
|
||||
run_pass(&mut scheduler).await.expect("first bounded fair pass");
|
||||
let mut per_bucket = BTreeMap::new();
|
||||
for table in &tables {
|
||||
if current(&store, table).await.is_some() {
|
||||
*per_bucket.entry(table.table_bucket.clone()).or_insert(0_usize) += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(per_bucket.values().sum::<usize>(), PERIODIC_MAINTENANCE_TABLES_PER_PASS);
|
||||
assert_eq!(per_bucket.len(), buckets.len(), "one bucket must not consume every admission");
|
||||
assert!(per_bucket.values().all(|count| *count <= 3));
|
||||
run_pass(&mut scheduler).await.expect("resume all retained cursors");
|
||||
for table in &tables {
|
||||
assert_eq!(
|
||||
current(&store, table)
|
||||
.await
|
||||
.expect("every table reached through retained cursors")
|
||||
.job
|
||||
.status,
|
||||
TableMetadataMaintenanceJobStatus::Queued
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_cancelled_or_expired_pass_does_not_start_discovery() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store, vec!["analytics".to_string()], StdDuration::from_secs(300));
|
||||
backend.reset_call_counts().await;
|
||||
let cancelled = CancellationToken::new();
|
||||
cancelled.cancel();
|
||||
scheduler
|
||||
.run_pass(&cancelled, Instant::now() + StdDuration::from_secs(60))
|
||||
.await
|
||||
.expect("cancelled pass");
|
||||
scheduler
|
||||
.run_pass(&CancellationToken::new(), Instant::now())
|
||||
.await
|
||||
.expect("expired pass");
|
||||
assert_eq!(backend.read_call_count().await, 0);
|
||||
assert_eq!(backend.list_call_count().await, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn periodic_queue_retains_inflight_publication_after_cancellation_and_deadline() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let first = seed_table(&store, &backend, "analytics", "sales", "a", Some(enabled_config())).await;
|
||||
let second = seed_table(&store, &backend, "analytics", "sales", "b", Some(enabled_config())).await;
|
||||
let namespace = Namespace::parse(&first.namespace).expect("fixture namespace");
|
||||
let name = IdentifierSegment::parse(&first.table).expect("fixture table");
|
||||
let path = TableCatalogObjectPaths::default().table_maintenance_current_job_path(
|
||||
"analytics",
|
||||
&namespace,
|
||||
&name,
|
||||
&first.table_id,
|
||||
);
|
||||
let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store.clone(), vec!["analytics".to_string()], StdDuration::from_secs(300));
|
||||
let cancellation = CancellationToken::new();
|
||||
let runner_cancellation = cancellation.clone();
|
||||
let deadline = Instant::now() + StdDuration::from_secs(30);
|
||||
let task = tokio::spawn(async move { scheduler.run_pass(&runner_cancellation, deadline).await });
|
||||
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.expect("queue reaches publication pause");
|
||||
cancellation.cancel();
|
||||
tokio::time::advance(StdDuration::from_secs(31)).await;
|
||||
assert!(
|
||||
!task.is_finished(),
|
||||
"cancellation must retain the mutating future until its write returns"
|
||||
);
|
||||
pause.release();
|
||||
task.await.expect("runner task").expect("in-flight queue completes");
|
||||
assert_eq!(
|
||||
current(&store, &first).await.expect("complete first publication").job.status,
|
||||
TableMetadataMaintenanceJobStatus::Queued
|
||||
);
|
||||
assert!(
|
||||
current(&store, &second).await.is_none(),
|
||||
"no operation starts after cancellation or admission deadline"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_atomic_guard_rejects_a_claim_changed_during_planning() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let pause = backend.pause_next_read(&table.table_bucket, &table.metadata_location).await;
|
||||
let claimant = store.clone();
|
||||
let observed = table.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
claimant
|
||||
.queue_periodic_table_maintenance(&observed, "first".to_string(), None)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.expect("first claimant starts unlocked planning");
|
||||
let winner = store
|
||||
.queue_periodic_table_maintenance(&table, "second".to_string(), None)
|
||||
.await
|
||||
.expect("second claimant wins while first is planning");
|
||||
let versions = object_versions(&backend).await;
|
||||
pause.release();
|
||||
assert!(matches!(
|
||||
task.await.expect("first claimant task"),
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
assert_eq!(
|
||||
current(&store, &table).await.expect("winning current job").job.job_id,
|
||||
winner.report.job.job_id
|
||||
);
|
||||
assert_eq!(object_versions(&backend).await, versions, "a stale claim must not publish any sidecar");
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(&table.table_bucket, &table.namespace, &table.table)
|
||||
.await
|
||||
.expect("read table"),
|
||||
Some(table)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_atomic_guard_rejects_changed_identity_config_and_partial_aliases() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let mut old_identity = table.clone();
|
||||
old_identity.table_id = uuid::Uuid::new_v4().to_string();
|
||||
let versions = object_versions(&backend).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.queue_periodic_table_maintenance(&old_identity, "periodic".to_string(), None)
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
assert_eq!(object_versions(&backend).await, versions);
|
||||
|
||||
for config in [
|
||||
TableMaintenanceConfig::default(),
|
||||
TableMaintenanceConfig {
|
||||
worker_paused: true,
|
||||
..enabled_config()
|
||||
},
|
||||
] {
|
||||
store
|
||||
.put_table_maintenance_config("analytics", "sales", "orders", config)
|
||||
.await
|
||||
.expect("change config after selection");
|
||||
let versions = object_versions(&backend).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.queue_periodic_table_maintenance(&table, "periodic".to_string(), None)
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
assert_eq!(
|
||||
object_versions(&backend).await,
|
||||
versions,
|
||||
"changed config must not cause Disabled or Paused control writes"
|
||||
);
|
||||
}
|
||||
store
|
||||
.put_table_maintenance_config("analytics", "sales", "orders", enabled_config())
|
||||
.await
|
||||
.expect("restore fixture config");
|
||||
let mut report = store
|
||||
.run_table_maintenance_scheduler_once("analytics", "sales", "orders", "fixture".to_string())
|
||||
.await
|
||||
.expect("seed full report")
|
||||
.report;
|
||||
report.job.status = TableMetadataMaintenanceJobStatus::Successful;
|
||||
report.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH));
|
||||
seed_report(&backend, &table, &report).await;
|
||||
let expected = table_maintenance_scheduler_job_summary(&report);
|
||||
report.job.status = TableMetadataMaintenanceJobStatus::Failed;
|
||||
let namespace = Namespace::parse(&table.namespace).expect("fixture namespace");
|
||||
let name = IdentifierSegment::parse(&table.table).expect("fixture table");
|
||||
let latest = TableCatalogObjectPaths::default().table_maintenance_latest_job_path(
|
||||
"analytics",
|
||||
&namespace,
|
||||
&name,
|
||||
&table.table_id,
|
||||
);
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&latest,
|
||||
serde_json::to_vec(&report).expect("serialize partial publication"),
|
||||
)
|
||||
.await;
|
||||
let versions = object_versions(&backend).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.queue_periodic_table_maintenance(&table, "periodic".to_string(), Some(&expected))
|
||||
.await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
assert_eq!(
|
||||
object_versions(&backend).await,
|
||||
versions,
|
||||
"partial report publication requires explicit recovery"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_empty_discovery_is_bounded_and_inactive_buckets_are_skipped() {
|
||||
for backing in backings() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), backing);
|
||||
store
|
||||
.put_table_bucket(test_bucket_entry("empty"))
|
||||
.await
|
||||
.expect("empty active bucket");
|
||||
let mut inactive = test_bucket_entry("inactive");
|
||||
inactive.state = TableCatalogEntryState::Deleted;
|
||||
store.put_table_bucket(inactive).await.expect("inactive fixture bucket");
|
||||
backend.reset_call_counts().await;
|
||||
let versions = object_versions(&backend).await;
|
||||
let mut scheduler = PeriodicMaintenanceScheduler::new(
|
||||
store,
|
||||
vec!["empty".to_string(), "inactive".to_string(), "missing".to_string()],
|
||||
StdDuration::from_secs(300),
|
||||
);
|
||||
run_pass(&mut scheduler)
|
||||
.await
|
||||
.expect("bounded discovery with no eligible tables");
|
||||
assert_eq!(object_versions(&backend).await, versions);
|
||||
assert!(
|
||||
backend.list_call_count().await <= PERIODIC_MAINTENANCE_DISCOVERY_STEPS_PER_PASS,
|
||||
"empty discovery must stop at the pass budget"
|
||||
);
|
||||
assert_eq!(scheduler.next_bucket, PERIODIC_MAINTENANCE_DISCOVERY_STEPS_PER_PASS % 3);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_queue_preserves_empty_object_page_continuations() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = TestStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
let table = seed_table(&store, &backend, "analytics", "sales", "orders", Some(enabled_config())).await;
|
||||
let prefix = TableCatalogObjectPaths::default().namespace_entries_prefix("analytics");
|
||||
for index in 0..TABLE_CATALOG_LIST_MAX_KEYS {
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &format!("{prefix}!non-entry-{index:04}"), b"fixture".to_vec())
|
||||
.await;
|
||||
}
|
||||
let first_page = store
|
||||
.list_namespaces_page("analytics", None, NonZeroUsize::MIN)
|
||||
.await
|
||||
.expect("bounded empty namespace page");
|
||||
assert!(first_page.entries.is_empty());
|
||||
assert!(first_page.next_cursor.is_some());
|
||||
let mut scheduler =
|
||||
PeriodicMaintenanceScheduler::new(store.clone(), vec!["analytics".to_string()], StdDuration::from_secs(300));
|
||||
run_pass(&mut scheduler).await.expect("continue past an empty page");
|
||||
assert_eq!(
|
||||
current(&store, &table)
|
||||
.await
|
||||
.expect("table after the empty page must be reached")
|
||||
.job
|
||||
.status,
|
||||
TableMetadataMaintenanceJobStatus::Queued
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::periodic::PeriodicMaintenanceScheduler;
|
||||
use crate::storage_api::startup::services::ServerContextSlot;
|
||||
use crate::storage_api::table::check_valid_bucket_name_strict;
|
||||
use crate::table_catalog::{
|
||||
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, ObjectTableCatalogStore, StrongTableCatalogStore,
|
||||
TableCatalogBackingMode, TableCatalogStoreError, TableCatalogStoreResult,
|
||||
};
|
||||
use std::{collections::BTreeSet, future::Future, sync::Arc, time::Duration};
|
||||
use tokio::{task::JoinHandle, time::Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const ENV_ENABLED: &str = "RUSTFS_TABLE_MAINTENANCE_SCHEDULER_ENABLED";
|
||||
const ENV_BUCKETS: &str = "RUSTFS_TABLE_MAINTENANCE_SCHEDULER_BUCKETS";
|
||||
const ENV_INTERVAL: &str = "RUSTFS_TABLE_MAINTENANCE_SCHEDULER_INTERVAL_SECONDS";
|
||||
const ADMISSION_BUDGET: Duration = Duration::from_secs(30);
|
||||
const LOG_COMPONENT: &str = "table_catalog";
|
||||
const LOG_SUBSYSTEM: &str = "maintenance";
|
||||
const EVENT_PERIODIC_SCHEDULER: &str = "table_maintenance_periodic_scheduler";
|
||||
|
||||
pub(crate) struct PeriodicMaintenanceConfig {
|
||||
buckets: Vec<String>,
|
||||
interval: Duration,
|
||||
backing: TableCatalogBackingMode,
|
||||
}
|
||||
|
||||
impl PeriodicMaintenanceConfig {
|
||||
pub(crate) fn from_env() -> TableCatalogStoreResult<Option<Self>> {
|
||||
Self::parse(
|
||||
|key| match std::env::var(key) {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(std::env::VarError::NotPresent) => Ok(None),
|
||||
Err(std::env::VarError::NotUnicode(_)) => {
|
||||
Err(TableCatalogStoreError::Invalid(format!("{key} must be valid UTF-8")))
|
||||
}
|
||||
},
|
||||
TableCatalogBackingMode::from_env,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse(
|
||||
read: impl Fn(&str) -> TableCatalogStoreResult<Option<String>>,
|
||||
backing: impl FnOnce() -> TableCatalogStoreResult<TableCatalogBackingMode>,
|
||||
) -> TableCatalogStoreResult<Option<Self>> {
|
||||
match read(ENV_ENABLED)?.as_deref().map(str::trim) {
|
||||
None | Some("false") => return Ok(None),
|
||||
Some("true") => {}
|
||||
_ => return Err(TableCatalogStoreError::Invalid(format!("{ENV_ENABLED} must be true or false"))),
|
||||
}
|
||||
let raw = read(ENV_BUCKETS)?
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid(format!("{ENV_BUCKETS} is required when scheduling is enabled")))?;
|
||||
if raw.len() > 2048 {
|
||||
return Err(TableCatalogStoreError::Invalid(format!("{ENV_BUCKETS} exceeds the allowlist size limit")));
|
||||
}
|
||||
let mut buckets = BTreeSet::new();
|
||||
for bucket in raw.split(',').map(str::trim) {
|
||||
check_valid_bucket_name_strict(bucket)
|
||||
.map_err(|_| TableCatalogStoreError::Invalid(format!("{ENV_BUCKETS} contains an invalid bucket name")))?;
|
||||
if !buckets.insert(bucket.to_string()) || buckets.len() > 32 {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"{ENV_BUCKETS} requires 1 to 32 distinct bucket names"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let interval = read(ENV_INTERVAL)?.unwrap_or_else(|| "300".to_string());
|
||||
let seconds = interval
|
||||
.trim()
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.filter(|seconds| (30..=86_400).contains(seconds))
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid(format!("{ENV_INTERVAL} must be between 30 and 86400")))?;
|
||||
Ok(Some(Self {
|
||||
buckets: buckets.into_iter().collect(),
|
||||
interval: Duration::from_secs(seconds),
|
||||
backing: backing()?,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns queueing until completion; never runs a worker or deletes table objects.
|
||||
pub(crate) struct PeriodicMaintenanceRuntime {
|
||||
cancellation: CancellationToken,
|
||||
task: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for PeriodicMaintenanceRuntime {
|
||||
fn drop(&mut self) {
|
||||
// Detach, never abort: in-flight sidecar writes retain their publication owner.
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl PeriodicMaintenanceRuntime {
|
||||
pub(crate) fn spawn(config: PeriodicMaintenanceConfig, server: Arc<ServerContextSlot>, shutdown: &CancellationToken) -> Self {
|
||||
let cancellation = shutdown.child_token();
|
||||
let task = tokio::spawn(run(config, server, cancellation.clone()));
|
||||
Self {
|
||||
cancellation,
|
||||
task: Some(task),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(mut self, grace: Duration) -> bool {
|
||||
self.cancellation.cancel();
|
||||
let Some(task) = self.task.take() else {
|
||||
return true;
|
||||
};
|
||||
let joined = matches!(tokio::time::timeout(grace, task).await, Ok(Ok(())));
|
||||
if !joined {
|
||||
warn!(
|
||||
event = EVENT_PERIODIC_SCHEDULER,
|
||||
component = LOG_COMPONENT,
|
||||
subsystem = LOG_SUBSYSTEM,
|
||||
state = "drain_unconfirmed",
|
||||
"Maintenance scheduler completion is unconfirmed; owned work was not aborted"
|
||||
);
|
||||
}
|
||||
joined
|
||||
}
|
||||
}
|
||||
|
||||
fn tick_delay(interval: Duration) -> Duration {
|
||||
// Delay after completion avoids catch-up bursts; jitter spreads node scans.
|
||||
interval + Duration::from_secs(rand::random_range(0..=interval.as_secs() / 10))
|
||||
}
|
||||
|
||||
async fn wait_for_tick(cancellation: &CancellationToken, delay: Duration) -> bool {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancellation.cancelled() => false,
|
||||
_ = tokio::time::sleep(delay) => true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_owned_pass<F: Future>(pass: F, cancellation: &CancellationToken, deadline: Instant) -> F::Output {
|
||||
tokio::pin!(pass);
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut pass => result,
|
||||
_ = cancellation.cancelled() => pass.await,
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
warn!(
|
||||
event = EVENT_PERIODIC_SCHEDULER, component = LOG_COMPONENT, subsystem = LOG_SUBSYSTEM,
|
||||
state = "draining", reason = "admission_deadline",
|
||||
"Maintenance scheduler stopped admitting work while its store call drains"
|
||||
);
|
||||
pass.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(config: PeriodicMaintenanceConfig, server: Arc<ServerContextSlot>, cancellation: CancellationToken) {
|
||||
let mut scheduler = None;
|
||||
let mut degraded = false;
|
||||
while wait_for_tick(&cancellation, tick_delay(config.interval)).await {
|
||||
if scheduler.is_none() {
|
||||
// Deferred IAM may not have installed this server yet. Do not use
|
||||
// another embedded server's ambient context as a fallback.
|
||||
let Some(context) = server.installed_app_context() else {
|
||||
continue;
|
||||
};
|
||||
let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
context.object_store(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
);
|
||||
let store = match config.backing {
|
||||
TableCatalogBackingMode::ObjectBacked => {
|
||||
ConfiguredTableCatalogStore::ObjectBacked(ObjectTableCatalogStore::new(backend))
|
||||
}
|
||||
TableCatalogBackingMode::DurableStrong => {
|
||||
ConfiguredTableCatalogStore::DurableStrong(StrongTableCatalogStore::new_requiring_snapshot(backend))
|
||||
}
|
||||
};
|
||||
scheduler = Some(PeriodicMaintenanceScheduler::new(store, config.buckets.clone(), config.interval));
|
||||
}
|
||||
let deadline = Instant::now() + ADMISSION_BUDGET;
|
||||
let Some(scheduler) = scheduler.as_mut() else {
|
||||
continue;
|
||||
};
|
||||
let result = finish_owned_pass(scheduler.run_pass(&cancellation, deadline), &cancellation, deadline).await;
|
||||
match result {
|
||||
Ok(()) if degraded => {
|
||||
info!(
|
||||
event = EVENT_PERIODIC_SCHEDULER,
|
||||
component = LOG_COMPONENT,
|
||||
subsystem = LOG_SUBSYSTEM,
|
||||
state = "recovered",
|
||||
"Maintenance scheduler pass recovered"
|
||||
);
|
||||
degraded = false;
|
||||
}
|
||||
Err(_) if !degraded => {
|
||||
warn!(
|
||||
event = EVENT_PERIODIC_SCHEDULER,
|
||||
component = LOG_COMPONENT,
|
||||
subsystem = LOG_SUBSYSTEM,
|
||||
state = "degraded",
|
||||
reason = "queue_pass_failed",
|
||||
"Maintenance scheduler could not complete a queueing pass"
|
||||
);
|
||||
degraded = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
fn config(values: &[(&str, &str)]) -> TableCatalogStoreResult<Option<PeriodicMaintenanceConfig>> {
|
||||
PeriodicMaintenanceConfig::parse(
|
||||
|key| {
|
||||
Ok(values
|
||||
.iter()
|
||||
.find(|(name, _)| *name == key)
|
||||
.map(|(_, value)| value.to_string()))
|
||||
},
|
||||
|| Ok(TableCatalogBackingMode::DurableStrong),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn periodic_config_is_opt_in_and_validates_limits() {
|
||||
assert!(config(&[]).unwrap().is_none());
|
||||
assert!(
|
||||
config(&[(ENV_ENABLED, "false"), (ENV_BUCKETS, "invalid/path")])
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for value in ["", "1", "yes", "TRUE"] {
|
||||
assert!(config(&[(ENV_ENABLED, value)]).is_err());
|
||||
}
|
||||
assert!(config(&[(ENV_ENABLED, "true")]).is_err());
|
||||
for buckets in ["", "foo,", "foo,foo", "../foo", "127.0.0.1", "Upper", ".minio.sys"] {
|
||||
assert!(config(&[(ENV_ENABLED, "true"), (ENV_BUCKETS, buckets)]).is_err(), "{buckets}");
|
||||
}
|
||||
let too_many = (0..33).map(|n| format!("bucket-{n}")).collect::<Vec<_>>().join(",");
|
||||
assert!(config(&[(ENV_ENABLED, "true"), (ENV_BUCKETS, &too_many)]).is_err());
|
||||
for interval in ["0", "29", "86401", "-1", "NaN", "18446744073709551616"] {
|
||||
assert!(config(&[(ENV_ENABLED, "true"), (ENV_BUCKETS, "foo"), (ENV_INTERVAL, interval)]).is_err());
|
||||
}
|
||||
let parsed = config(&[(ENV_ENABLED, "true"), (ENV_BUCKETS, "bbb, aaa")]).unwrap().unwrap();
|
||||
assert_eq!(parsed.buckets, ["aaa", "bbb"]);
|
||||
assert_eq!(parsed.interval, Duration::from_secs(300));
|
||||
for _ in 0..100 {
|
||||
assert!((Duration::from_secs(300)..=Duration::from_secs(330)).contains(&tick_delay(parsed.interval)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn periodic_tick_waits_and_cancels_without_work() {
|
||||
let cancellation = CancellationToken::new();
|
||||
let start = Instant::now();
|
||||
assert!(wait_for_tick(&cancellation, Duration::from_secs(300)).await);
|
||||
assert_eq!(Instant::now() - start, Duration::from_secs(300));
|
||||
cancellation.cancel();
|
||||
assert!(!wait_for_tick(&cancellation, Duration::from_secs(300)).await);
|
||||
assert_eq!(Instant::now() - start, Duration::from_secs(300));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn periodic_deadline_and_shutdown_retain_owned_work() {
|
||||
for cancel in [false, true] {
|
||||
let cancellation = CancellationToken::new();
|
||||
let completed = Arc::new(AtomicBool::new(false));
|
||||
let (entered, started) = tokio::sync::oneshot::channel();
|
||||
let (release, resume) = tokio::sync::oneshot::channel();
|
||||
let done = completed.clone();
|
||||
let token = cancellation.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
finish_owned_pass(
|
||||
async move {
|
||||
entered.send(()).unwrap();
|
||||
resume.await.unwrap();
|
||||
done.store(true, Ordering::SeqCst);
|
||||
},
|
||||
&token,
|
||||
Instant::now() + Duration::from_secs(30),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
started.await.unwrap();
|
||||
if cancel {
|
||||
cancellation.cancel();
|
||||
} else {
|
||||
tokio::time::advance(Duration::from_secs(31)).await;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
assert!(!completed.load(Ordering::SeqCst));
|
||||
release.send(()).unwrap();
|
||||
task.await.unwrap();
|
||||
assert!(completed.load(Ordering::SeqCst));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn periodic_shutdown_reports_incomplete_drain_without_aborting() {
|
||||
let completed = Arc::new(AtomicBool::new(false));
|
||||
let done = completed.clone();
|
||||
let (release, resume) = tokio::sync::oneshot::channel();
|
||||
let (finished, observed) = tokio::sync::oneshot::channel();
|
||||
let runtime = PeriodicMaintenanceRuntime {
|
||||
cancellation: CancellationToken::new(),
|
||||
task: Some(tokio::spawn(async move {
|
||||
resume.await.unwrap();
|
||||
done.store(true, Ordering::SeqCst);
|
||||
finished.send(()).unwrap();
|
||||
})),
|
||||
};
|
||||
assert!(!runtime.shutdown(Duration::from_secs(30)).await);
|
||||
assert!(!completed.load(Ordering::SeqCst));
|
||||
release.send(()).unwrap();
|
||||
observed.await.unwrap();
|
||||
assert!(completed.load(Ordering::SeqCst));
|
||||
let runtime = PeriodicMaintenanceRuntime {
|
||||
cancellation: CancellationToken::new(),
|
||||
task: Some(tokio::spawn(async {})),
|
||||
};
|
||||
assert!(runtime.shutdown(Duration::from_secs(30)).await);
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,11 @@ pub(super) use strong::{
|
||||
};
|
||||
pub(crate) use strong::{StrongTableCatalogRuntime, StrongTableCatalogStore};
|
||||
|
||||
struct PeriodicTableMaintenanceQueueGuard<'a> {
|
||||
table_id: &'a str,
|
||||
current_job: Option<&'a TableMaintenanceSchedulerJobSummary>,
|
||||
}
|
||||
|
||||
pub(in crate::table_catalog) fn catalog_lock_acquisition_error(
|
||||
operation: &str,
|
||||
err: rustfs_lock::LockError,
|
||||
@@ -1924,6 +1929,43 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn queue_periodic_table_maintenance(
|
||||
&self,
|
||||
entry: &TableEntry,
|
||||
scheduler_id: String,
|
||||
expected_job: Option<&TableMaintenanceSchedulerJobSummary>,
|
||||
) -> TableCatalogStoreResult<TableMaintenanceSchedulerRunResult> {
|
||||
let guard = PeriodicTableMaintenanceQueueGuard {
|
||||
table_id: &entry.table_id,
|
||||
current_job: expected_job,
|
||||
};
|
||||
match self {
|
||||
Self::ObjectBacked(store) => {
|
||||
store
|
||||
.run_table_maintenance_scheduler_once_with_guard_at(
|
||||
&entry.table_bucket,
|
||||
&entry.namespace,
|
||||
&entry.table,
|
||||
scheduler_id,
|
||||
OffsetDateTime::now_utc(),
|
||||
Some(&guard),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Self::DurableStrong(store) => {
|
||||
store
|
||||
.run_table_maintenance_scheduler_once_with_guard(
|
||||
&entry.table_bucket,
|
||||
&entry.namespace,
|
||||
&entry.table,
|
||||
scheduler_id,
|
||||
Some(&guard),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_table_maintenance_quarantine_operation(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
|
||||
@@ -3564,6 +3564,19 @@ where
|
||||
table: &str,
|
||||
scheduler_id: String,
|
||||
now: OffsetDateTime,
|
||||
) -> TableCatalogStoreResult<TableMaintenanceSchedulerRunResult> {
|
||||
self.run_table_maintenance_scheduler_once_with_guard_at(table_bucket, namespace, table, scheduler_id, now, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn run_table_maintenance_scheduler_once_with_guard_at(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
scheduler_id: String,
|
||||
now: OffsetDateTime,
|
||||
periodic_guard: Option<&PeriodicTableMaintenanceQueueGuard<'_>>,
|
||||
) -> TableCatalogStoreResult<TableMaintenanceSchedulerRunResult> {
|
||||
let namespace = parse_namespace_for_store(namespace)?;
|
||||
let table = parse_table_for_store(table)?;
|
||||
@@ -3583,6 +3596,19 @@ where
|
||||
let effective = self
|
||||
.get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry)
|
||||
.await?;
|
||||
if let Some(periodic_guard) = periodic_guard {
|
||||
self.validate_periodic_table_maintenance_queue_guard(
|
||||
TableMaintenancePreflightContext {
|
||||
table_bucket,
|
||||
namespace: &namespace,
|
||||
table: &table,
|
||||
entry: &entry,
|
||||
},
|
||||
&effective,
|
||||
periodic_guard,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
self.table_metadata_maintenance_scheduler_preflight(
|
||||
TableMaintenancePreflightContext {
|
||||
table_bucket,
|
||||
@@ -3629,6 +3655,19 @@ where
|
||||
let effective = self
|
||||
.get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry)
|
||||
.await?;
|
||||
if let Some(periodic_guard) = periodic_guard {
|
||||
self.validate_periodic_table_maintenance_queue_guard(
|
||||
TableMaintenancePreflightContext {
|
||||
table_bucket,
|
||||
namespace: &namespace,
|
||||
table: &table,
|
||||
entry: &entry,
|
||||
},
|
||||
&effective,
|
||||
periodic_guard,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
match self
|
||||
.table_metadata_maintenance_scheduler_preflight(
|
||||
TableMaintenancePreflightContext {
|
||||
@@ -3936,6 +3975,61 @@ where
|
||||
Ok(reports)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_periodic_table_maintenance_queue_guard(
|
||||
&self,
|
||||
context: TableMaintenancePreflightContext<'_>,
|
||||
effective: &TableMaintenanceEffectiveConfig,
|
||||
expected: &PeriodicTableMaintenanceQueueGuard<'_>,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if context.entry.table_id != expected.table_id {
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table identity changed before periodic maintenance claim".to_string(),
|
||||
));
|
||||
}
|
||||
if !effective.config.background_enabled || effective.config.worker_paused {
|
||||
return Err(TableCatalogStoreError::Conflict("periodic maintenance is disabled or paused".to_string()));
|
||||
}
|
||||
let current = self
|
||||
.get_table_metadata_maintenance_report_for_entry_unlocked(
|
||||
context.table_bucket,
|
||||
context.namespace,
|
||||
context.table,
|
||||
&context.entry.table_id,
|
||||
MAINTENANCE_JOB_ALIAS_CURRENT,
|
||||
)
|
||||
.await?;
|
||||
if current.as_ref().map(table_maintenance_scheduler_job_summary).as_ref() != expected.current_job
|
||||
|| current
|
||||
.as_ref()
|
||||
.is_some_and(|report| report.job.status != TableMetadataMaintenanceJobStatus::Successful)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"maintenance job changed or requires explicit operator action before periodic scheduling".to_string(),
|
||||
));
|
||||
}
|
||||
let latest = self
|
||||
.get_table_metadata_maintenance_report_for_entry_unlocked(
|
||||
context.table_bucket,
|
||||
context.namespace,
|
||||
context.table,
|
||||
&context.entry.table_id,
|
||||
MAINTENANCE_JOB_ALIAS_LATEST,
|
||||
)
|
||||
.await?;
|
||||
// Report publication writes latest before current. A partial write must
|
||||
// not let an automatic pass replace an unresolved job or quarantine.
|
||||
if latest != current
|
||||
|| current
|
||||
.as_ref()
|
||||
.is_some_and(|report| report.job.quarantine_enabled && report.job.quarantined_object_count > 0)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"maintenance report publication or quarantine requires recovery before periodic scheduling".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn table_metadata_maintenance_scheduler_preflight(
|
||||
&self,
|
||||
context: TableMaintenancePreflightContext<'_>,
|
||||
@@ -5153,7 +5247,7 @@ where
|
||||
);
|
||||
}
|
||||
}
|
||||
let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok();
|
||||
let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok();
|
||||
let current_metadata_location = entry.metadata_location.clone();
|
||||
let retained_metadata_locations = retained.into_iter().collect::<Vec<_>>();
|
||||
let object_reports = metadata_maintenance_object_reports(maintenance_reasons);
|
||||
@@ -5677,7 +5771,7 @@ where
|
||||
"current metadata location changed before maintenance delete".to_string(),
|
||||
));
|
||||
}
|
||||
let warehouse_object_prefix = table_warehouse_object_prefix(&entry).ok();
|
||||
let warehouse_object_prefix = table_warehouse_object_prefix(entry).ok();
|
||||
|
||||
let Some(current_metadata) = read_table_metadata_value(&self.backend, table_bucket, &entry.metadata_location).await?
|
||||
else {
|
||||
|
||||
@@ -2199,6 +2199,18 @@ where
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
scheduler_id: String,
|
||||
) -> TableCatalogStoreResult<TableMaintenanceSchedulerRunResult> {
|
||||
self.run_table_maintenance_scheduler_once_with_guard(table_bucket, namespace, table, scheduler_id, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn run_table_maintenance_scheduler_once_with_guard(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
namespace: &str,
|
||||
table: &str,
|
||||
scheduler_id: String,
|
||||
periodic_guard: Option<&PeriodicTableMaintenanceQueueGuard<'_>>,
|
||||
) -> TableCatalogStoreResult<TableMaintenanceSchedulerRunResult> {
|
||||
let namespace = parse_namespace_for_store(namespace)?;
|
||||
let table = parse_table_for_store(table)?;
|
||||
@@ -2223,6 +2235,20 @@ where
|
||||
let effective = sidecar
|
||||
.get_effective_table_maintenance_config_for_entry_unlocked(table_bucket, &namespace, &table, &entry)
|
||||
.await?;
|
||||
if let Some(periodic_guard) = periodic_guard {
|
||||
sidecar
|
||||
.validate_periodic_table_maintenance_queue_guard(
|
||||
TableMaintenancePreflightContext {
|
||||
table_bucket,
|
||||
namespace: &namespace,
|
||||
table: &table,
|
||||
entry: &entry,
|
||||
},
|
||||
&effective,
|
||||
periodic_guard,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let preflight = sidecar
|
||||
.table_metadata_maintenance_scheduler_preflight_with_publication(
|
||||
TableMaintenancePreflightContext {
|
||||
@@ -2281,6 +2307,20 @@ where
|
||||
if effective.config != planned_effective.config || effective.source != planned_effective.source {
|
||||
return Err(TableCatalogStoreError::Conflict("maintenance config changed during planning".to_string()));
|
||||
}
|
||||
if let Some(periodic_guard) = periodic_guard {
|
||||
sidecar
|
||||
.validate_periodic_table_maintenance_queue_guard(
|
||||
TableMaintenancePreflightContext {
|
||||
table_bucket,
|
||||
namespace: &namespace,
|
||||
table: &table,
|
||||
entry: &entry,
|
||||
},
|
||||
&effective,
|
||||
periodic_guard,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
match sidecar
|
||||
.table_metadata_maintenance_scheduler_preflight_with_publication(
|
||||
TableMaintenancePreflightContext {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/bin/sh
|
||||
# Reject the distrusted emSign ECC Root CA - G3 in native trust stores.
|
||||
set -eu
|
||||
|
||||
bundle=${1:?Usage: check_ca_bundle.sh BUNDLE CERT_DIRECTORY}
|
||||
cert_directory=${2:?Usage: check_ca_bundle.sh BUNDLE CERT_DIRECTORY}
|
||||
distrusted_sha256=86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT HUP INT TERM
|
||||
|
||||
check_file() {
|
||||
# Hash DER, not subject text or PEM formatting. Also inspect hashed links:
|
||||
# native trust consumers may use the directory rather than the bundle.
|
||||
awk '
|
||||
/-----BEGIN CERTIFICATE-----/ { if (inside) exit 1; inside=1; cert=""; next }
|
||||
/-----END CERTIFICATE-----/ { if (!inside) exit 1; print cert; inside=0; next }
|
||||
inside { gsub(/[[:space:]]/, ""); cert=cert $0 }
|
||||
END { if (inside) exit 1 }
|
||||
' "$1" > "$scratch/certificates"
|
||||
while IFS= read -r encoded; do
|
||||
printf '%s' "$encoded" | base64 -d > "$scratch/certificate.der"
|
||||
if [ ! -s "$scratch/certificate.der" ]; then
|
||||
echo "Empty certificate in $1" >&2
|
||||
return 1
|
||||
fi
|
||||
digest=$(sha256sum "$scratch/certificate.der")
|
||||
digest=${digest%% *}
|
||||
if [ "$digest" = "$distrusted_sha256" ]; then
|
||||
echo "Distrusted emSign ECC Root CA - G3 found in $1" >&2
|
||||
return 1
|
||||
fi
|
||||
done < "$scratch/certificates"
|
||||
}
|
||||
|
||||
check_file "$bundle"
|
||||
if [ ! -s "$scratch/certificates" ]; then
|
||||
echo "CA bundle contains no certificates: $bundle" >&2
|
||||
exit 1
|
||||
fi
|
||||
find -L "$cert_directory" -type f > "$scratch/files"
|
||||
while IFS= read -r file; do
|
||||
check_file "$file"
|
||||
done < "$scratch/files"
|
||||
@@ -0,0 +1,15 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
|
||||
EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
|
||||
bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
|
||||
RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
|
||||
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
|
||||
b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
|
||||
djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
|
||||
WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
|
||||
fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
|
||||
zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
|
||||
hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
|
||||
CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
|
||||
+JbNR6iC8hZVdyR+EhCVBCyj
|
||||
-----END CERTIFICATE-----
|
||||
@@ -200,7 +200,7 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
||||
"status": "controlled-run-once-supported",
|
||||
"roadmap_area": "maintenance-worker",
|
||||
"catalog_endpoint": "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/maintenance/scheduler",
|
||||
"expected_behavior": "background-enabled maintenance can be queued by the scheduler run endpoint, claimed by the worker run endpoint, and inspected through the scheduler status endpoint with disabled/paused state, queued-job handoff, current-job backpressure, retry deferral, quarantine boundary, audit timeline, per-job audit events, lease expiry recovery, heartbeat updates, and operator quarantine inspect/release/retry/abandon actions; built-in periodic scheduling is not claimed",
|
||||
"expected_behavior": "background-enabled maintenance can be queued by the scheduler run endpoint, claimed by the worker run endpoint, and inspected through the scheduler status endpoint with disabled/paused state, queued-job handoff, current-job backpressure, retry deferral, quarantine boundary, audit timeline, per-job audit events, lease expiry recovery, heartbeat updates, and operator quarantine inspect/release/retry/abandon actions; opt-in periodic admission can queue jobs for allowlisted buckets, while workers still require an explicit operator invocation",
|
||||
},
|
||||
{
|
||||
"capability": "manifest-data-reachability-cleanup",
|
||||
@@ -212,13 +212,13 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
||||
"capability": "snapshot-expiration",
|
||||
"status": "manual-maintenance-supported",
|
||||
"roadmap_area": "snapshot-maintenance",
|
||||
"expected_behavior": "metadata maintenance can report snapshot expiration plans and manually commit safe snapshot expiration through the catalog; background scheduling and manifest/data cleanup are not claimed",
|
||||
"expected_behavior": "metadata maintenance can report snapshot expiration plans and manually commit safe snapshot expiration through the catalog; automatic snapshot expiration and background manifest/data cleanup are not claimed",
|
||||
},
|
||||
{
|
||||
"capability": "compaction-rewrite",
|
||||
"status": "controlled-run-once-supported",
|
||||
"roadmap_area": "snapshot-maintenance",
|
||||
"expected_behavior": "metadata maintenance can plan binpack candidates and commit a safe partition-local and sort-order-preserving Parquet rewrite through the catalog; manifests with position or equality delete files produce machine-readable row-level planning and fail closed before rewrite; built-in periodic scheduling, delete-file rewrite, and row-level compaction execution are not claimed",
|
||||
"expected_behavior": "metadata maintenance can plan binpack candidates and commit a safe partition-local and sort-order-preserving Parquet rewrite through the catalog; manifests with position or equality delete files produce machine-readable row-level planning and fail closed before rewrite; automatic compaction execution, delete-file rewrite, and row-level compaction execution are not claimed",
|
||||
},
|
||||
{
|
||||
"capability": "row-level-delete-update-merge",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
repo_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT HUP INT TERM
|
||||
mkdir "$scratch/certs"
|
||||
# A public root from webpki-roots 1.0.9, not a private key or active trust anchor.
|
||||
distrusted="$repo_root/scripts/fixtures/distrusted-emsign-g3.pem"
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=ca-bundle-test \
|
||||
-keyout "$scratch/key.pem" -out "$scratch/valid.pem" >/dev/null 2>&1
|
||||
cp "$scratch/valid.pem" "$scratch/bundle.pem"
|
||||
check() {
|
||||
sh "$repo_root/scripts/check_ca_bundle.sh" "$scratch/bundle.pem" "$scratch/certs"
|
||||
}
|
||||
reject() {
|
||||
if check > "$scratch/output" 2>&1; then
|
||||
echo "Expected CA rejection: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
check
|
||||
cat "$distrusted" >> "$scratch/bundle.pem"
|
||||
reject 'G3 after a valid certificate in the bundle'
|
||||
cp "$scratch/valid.pem" "$scratch/bundle.pem"
|
||||
cp "$distrusted" "$scratch/certs/renamed-root.pem"
|
||||
reject 'G3 in directory with an unrelated name'
|
||||
rm "$scratch/certs/renamed-root.pem"
|
||||
ln -s "$distrusted" "$scratch/certs/12345678.0"
|
||||
reject 'G3 through a hashed trust-directory symlink'
|
||||
rm "$scratch/certs/12345678.0"
|
||||
# Equivalent PEM encodings must not defeat DER-based detection.
|
||||
printf '%s\n' '-----BEGIN CERTIFICATE-----' > "$scratch/bundle.pem"
|
||||
sed '/-----/d' "$distrusted" | tr -d '\n' >> "$scratch/bundle.pem"
|
||||
printf '\n%s\n' '-----END CERTIFICATE-----' >> "$scratch/bundle.pem"
|
||||
reject 'G3 with unwrapped PEM body'
|
||||
printf '%s\n' '-----BEGIN CERTIFICATE-----' > "$scratch/bundle.pem"
|
||||
reject 'truncated bundle'
|
||||
printf '%s\n' '-----BEGIN CERTIFICATE-----' '-----END CERTIFICATE-----' > "$scratch/bundle.pem"
|
||||
reject 'empty PEM body'
|
||||
: > "$scratch/bundle.pem"
|
||||
reject 'empty bundle'
|
||||
cp "$scratch/valid.pem" "$scratch/bundle.pem"
|
||||
check
|
||||
printf '%s\n' 'CA bundle regression checks passed'
|
||||
@@ -45,6 +45,32 @@ def archive(path, members, binary):
|
||||
|
||||
|
||||
class ReleaseBuildTests(unittest.TestCase):
|
||||
def test_security_release_console_pin(self):
|
||||
script = shell_body(STEPS['Download static console assets'])
|
||||
self.assertIn('releases/tags/v0.1.37', script)
|
||||
self.assertNotIn('console/releases/latest', script)
|
||||
parser = re.search(r"<<'PY'\n(.*?)\nPY", script, re.S)[1]
|
||||
expected_digest = 'sha256:e6ff2c7e7695de824ec4cda91d8611d1aba5d8462e6809592c92a50b23f5b9ab'
|
||||
asset = {'name': 'rustfs-console-v0.1.37.zip', 'digest': expected_digest,
|
||||
'browser_download_url': 'https://github.com/rustfs/console/releases/download/v0.1.37/rustfs-console-v0.1.37.zip'}
|
||||
cases = [
|
||||
('exact pin', 'v0.1.37', asset, True),
|
||||
('different tag', 'v0.1.38', asset, False),
|
||||
('different name', 'v0.1.37', {**asset, 'name': 'rustfs-console-v0.1.38.zip'}, False),
|
||||
('different digest', 'v0.1.37', {**asset, 'digest': 'sha256:' + '0' * 64}, False),
|
||||
('missing URL', 'v0.1.37', {**asset, 'browser_download_url': ''}, False),
|
||||
]
|
||||
for name, tag, entry, accepted in cases:
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
|
||||
metadata = Path(directory) / 'release.json'
|
||||
metadata.write_text(json.dumps({'tag_name': tag, 'assets': [entry]}))
|
||||
result = subprocess.run([sys.executable, '-c', parser, str(metadata)],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode == 0, accepted, result.stderr)
|
||||
if accepted:
|
||||
self.assertEqual(result.stdout.split(),
|
||||
['v0.1.37', asset['browser_download_url'], expected_digest.split(':')[1]])
|
||||
|
||||
def test_every_matrix_target_builds_only_server(self):
|
||||
mock_cargo = r'''
|
||||
cargo() {
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
/Cargo.lock
|
||||
/target/
|
||||
Vendored
-265
@@ -1,265 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
## [2.1.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.1.0...lib-2.1.1) (2026-07-21)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* avoid slice construction in address validation ([#569](https://github.com/grafana/pyroscope-rs/issues/569)) ([d6f45fe](https://github.com/grafana/pyroscope-rs/commit/d6f45fef1443d26c64a40721013ecfa7b1d3eccd))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** lock file maintenance ([#570](https://github.com/grafana/pyroscope-rs/issues/570)) ([c37f52e](https://github.com/grafana/pyroscope-rs/commit/c37f52e71719da9ddb4a32ac8386a4c94534c805))
|
||||
|
||||
## [2.1.0](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.6...lib-2.1.0) (2026-07-07)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add a method to set the timer's ticker in PyroscopeAgent ([#557](https://github.com/grafana/pyroscope-rs/issues/557)) ([cc13d38](https://github.com/grafana/pyroscope-rs/commit/cc13d38cb5e84017a4f5e229648457b790bd6a1a))
|
||||
* add otel.scope.name, otel.scope.version, process.runtime.name, process.runtime.version labels ([#562](https://github.com/grafana/pyroscope-rs/issues/562)) ([32f076e](https://github.com/grafana/pyroscope-rs/commit/32f076efd45aa2f3f203eb0bf8b03f222cb40648))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **security/unknown/:** update security memmap2 to v0.9.11 [security] ([#558](https://github.com/grafana/pyroscope-rs/issues/558)) ([d4d83f8](https://github.com/grafana/pyroscope-rs/commit/d4d83f8d350d927739c79a53bcfd7cd0115c77a4))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* bump nix ([#566](https://github.com/grafana/pyroscope-rs/issues/566)) ([1c02f30](https://github.com/grafana/pyroscope-rs/commit/1c02f3058135d1db06262ca57ead33fdc1593e6f))
|
||||
* **deps:** lock file maintenance ([#555](https://github.com/grafana/pyroscope-rs/issues/555)) ([3917b99](https://github.com/grafana/pyroscope-rs/commit/3917b99d6de4f4d6cec078aba2ec7b6f4e7044c7))
|
||||
|
||||
## [2.0.6](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.5...lib-2.0.6) (2026-05-28)
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** vendor pprofrs ([#552](https://github.com/grafana/pyroscope-rs/issues/552)) ([6684026](https://github.com/grafana/pyroscope-rs/commit/6684026ba0ea4fc997e4869703d671f23ee377a7))
|
||||
|
||||
|
||||
### Continuous Integration
|
||||
|
||||
* remove pr-title-check workflow ([#549](https://github.com/grafana/pyroscope-rs/issues/549)) ([e4739e4](https://github.com/grafana/pyroscope-rs/commit/e4739e4bfa280e771fbd932f39d53faa599718ba))
|
||||
|
||||
## [2.0.5](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.4...lib-2.0.5) (2026-05-27)
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** Remove unnecessary framehop dep ([#550](https://github.com/grafana/pyroscope-rs/issues/550)) ([d9c0e02](https://github.com/grafana/pyroscope-rs/commit/d9c0e02c7386b3a55e0270083c1135cac0e3ce0e))
|
||||
|
||||
## [2.0.4](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.3...lib-2.0.4) (2026-05-20)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* do not recreate profiling guard, just reset the data ([#518](https://github.com/grafana/pyroscope-rs/issues/518)) ([b1fe114](https://github.com/grafana/pyroscope-rs/commit/b1fe1141ee0d86caa59db383298c818763f3fa95))
|
||||
* remove kindasafe CI leftovers ([#521](https://github.com/grafana/pyroscope-rs/issues/521)) ([f8bc0f3](https://github.com/grafana/pyroscope-rs/commit/f8bc0f31c3b775b9d0e97fb412af76059f0819ec))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** bump openssl from 0.10.79 to 0.10.80 ([#548](https://github.com/grafana/pyroscope-rs/issues/548)) ([f3ed911](https://github.com/grafana/pyroscope-rs/commit/f3ed911e8b179d6fd63eacc2a793e884f126dfa0))
|
||||
* **deps:** lock file maintenance ([#535](https://github.com/grafana/pyroscope-rs/issues/535)) ([9554564](https://github.com/grafana/pyroscope-rs/commit/955456438ea768adfe0a89cd1ef7da22e7974bcd))
|
||||
* **deps:** lock file maintenance ([#541](https://github.com/grafana/pyroscope-rs/issues/541)) ([3abc7b6](https://github.com/grafana/pyroscope-rs/commit/3abc7b6d0c01f420a16cd2d08f44f10424c3ef3c))
|
||||
* **deps:** lock file maintenance ([#547](https://github.com/grafana/pyroscope-rs/issues/547)) ([cd30ebf](https://github.com/grafana/pyroscope-rs/commit/cd30ebfb07bfbf243125df53f051a2f2d1dd5d63))
|
||||
* **deps:** pin rust docker tag to a9cfb75 ([#544](https://github.com/grafana/pyroscope-rs/issues/544)) ([ea14091](https://github.com/grafana/pyroscope-rs/commit/ea140911e4b8bbabc8c42fbb2c7d43e0384ec4fe))
|
||||
* **deps:** remove names crate dependency ([#527](https://github.com/grafana/pyroscope-rs/issues/527)) ([5893aac](https://github.com/grafana/pyroscope-rs/commit/5893aacb5bc3660dcd269b2fee33847f1fe1ab82))
|
||||
* **deps:** update rust crate libc to v0.2.186 ([#533](https://github.com/grafana/pyroscope-rs/issues/533)) ([a7468c1](https://github.com/grafana/pyroscope-rs/commit/a7468c1db5d7955dfa0eb702a8b0371775ff2350))
|
||||
* **deps:** update rust crate reqwest to v0.13.3 ([#539](https://github.com/grafana/pyroscope-rs/issues/539)) ([b8c6fbb](https://github.com/grafana/pyroscope-rs/commit/b8c6fbb97cc5f3b22185be35c9e19e72237b912e))
|
||||
* **deps:** update rust:trixie docker digest to a9cfb75 ([#538](https://github.com/grafana/pyroscope-rs/issues/538)) ([bff1d5d](https://github.com/grafana/pyroscope-rs/commit/bff1d5d98f2a81d7a291c257bfb8291b41ffc8e5))
|
||||
* **renovate:** vulnerabilities-only with profiling deps allowlist ([#540](https://github.com/grafana/pyroscope-rs/issues/540)) ([4aa8e56](https://github.com/grafana/pyroscope-rs/commit/4aa8e5656472a08ddea21cb2128b85203920243b))
|
||||
|
||||
|
||||
### Continuous Integration
|
||||
|
||||
* add ARM cross-compilation test job ([#536](https://github.com/grafana/pyroscope-rs/issues/536)) ([54bb03f](https://github.com/grafana/pyroscope-rs/commit/54bb03f6996a74ad98cc75bdd08d790bce61efa3))
|
||||
* auto-publish to crates.io after release-please PR merge ([#529](https://github.com/grafana/pyroscope-rs/issues/529)) ([a4f76b6](https://github.com/grafana/pyroscope-rs/commit/a4f76b6f9de6b6c35e4d947aba4dbfe49d8c019a))
|
||||
* Potential fix for code scanning alert no. 14: Workflow does not contain permissions ([#542](https://github.com/grafana/pyroscope-rs/issues/542)) ([51ac4b7](https://github.com/grafana/pyroscope-rs/commit/51ac4b78e4a8cc0c8130984b52181fce12e3589f))
|
||||
|
||||
## [2.0.3](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.2...lib-2.0.3) (2026-04-26)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **epoll:** use libc::c_long for itimerspec fields ([#532](https://github.com/grafana/pyroscope-rs/issues/532)) ([093406e](https://github.com/grafana/pyroscope-rs/commit/093406e667175e5bc28ca7a1c5efe62be6b9b4a0))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** update googleapis/release-please-action action to v5 ([#530](https://github.com/grafana/pyroscope-rs/issues/530)) ([6c97b9d](https://github.com/grafana/pyroscope-rs/commit/6c97b9d414c7a1e26b07334a1cf0922e1c2cd2c9))
|
||||
|
||||
## [2.0.2](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.1...lib-2.0.2) (2026-04-24)
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** bump rustls-webpki from 0.103.12 to 0.103.13 ([#524](https://github.com/grafana/pyroscope-rs/issues/524)) ([935384d](https://github.com/grafana/pyroscope-rs/commit/935384dc4d3479a1a12df2502898a8813b775884))
|
||||
* **deps:** lock file maintenance ([#522](https://github.com/grafana/pyroscope-rs/issues/522)) ([4bd95bb](https://github.com/grafana/pyroscope-rs/commit/4bd95bb47f84f3aa6db6aa22173a7d74f774ab45))
|
||||
* **deps:** update dependency ruby to v4.0.3 ([#523](https://github.com/grafana/pyroscope-rs/issues/523)) ([e126aa2](https://github.com/grafana/pyroscope-rs/commit/e126aa24517ba6efea62302e465ea736e4cef54c))
|
||||
* **deps:** update rust:trixie docker digest to e4f09e8 ([#515](https://github.com/grafana/pyroscope-rs/issues/515)) ([f450327](https://github.com/grafana/pyroscope-rs/commit/f450327b1901f43d97d96eee5e0d73ccb64aa768))
|
||||
|
||||
## [2.0.1](https://github.com/grafana/pyroscope-rs/compare/lib-2.0.0...lib-2.0.1) (2026-04-19)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* revert 0976d999e4a50c14459b8a0b39e72434be6d4bfc ([#483](https://github.com/grafana/pyroscope-rs/issues/483)) ([ad3d032](https://github.com/grafana/pyroscope-rs/commit/ad3d03217a14a18868b8ba21f3dbf4ed844d759a))
|
||||
|
||||
|
||||
### Miscellaneous Chores
|
||||
|
||||
* **deps:** lock file maintenance ([#467](https://github.com/grafana/pyroscope-rs/issues/467)) ([09411da](https://github.com/grafana/pyroscope-rs/commit/09411daca91120c4863983273863c55b02b4dd89))
|
||||
* **deps:** lock file maintenance ([#478](https://github.com/grafana/pyroscope-rs/issues/478)) ([447d8ee](https://github.com/grafana/pyroscope-rs/commit/447d8eef3621be5b9b09b0b9eca3968bca081c2a))
|
||||
* **deps:** lock file maintenance ([#498](https://github.com/grafana/pyroscope-rs/issues/498)) ([4072a04](https://github.com/grafana/pyroscope-rs/commit/4072a04626abd98aa18bc6cda37fb0ccc0803f59))
|
||||
* **deps:** lock file maintenance ([#502](https://github.com/grafana/pyroscope-rs/issues/502)) ([bef3bc3](https://github.com/grafana/pyroscope-rs/commit/bef3bc3ec08b57b74d5d177204118492cdaa50b2))
|
||||
* **deps:** pin dependencies ([#427](https://github.com/grafana/pyroscope-rs/issues/427)) ([a46830a](https://github.com/grafana/pyroscope-rs/commit/a46830ac9a75a7634d9cc1ecc514a21b5c0f0603))
|
||||
* **deps:** pin rust docker tag to e8e2bb5 ([#501](https://github.com/grafana/pyroscope-rs/issues/501)) ([c177d5b](https://github.com/grafana/pyroscope-rs/commit/c177d5b8f28041d3956851d93e61be3aebc6d975))
|
||||
* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#480](https://github.com/grafana/pyroscope-rs/issues/480)) ([c4f2e7d](https://github.com/grafana/pyroscope-rs/commit/c4f2e7dd1b1234f8870ce0f858f92542aef37c2e))
|
||||
* **deps:** update dtolnay/rust-toolchain digest to 3c5f7ea ([#500](https://github.com/grafana/pyroscope-rs/issues/500)) ([fc21bda](https://github.com/grafana/pyroscope-rs/commit/fc21bda959bdf5d5d5a44f559a2b9de8072d2fba))
|
||||
* **deps:** update googleapis/release-please-action digest to 5c625bf ([#509](https://github.com/grafana/pyroscope-rs/issues/509)) ([c347e17](https://github.com/grafana/pyroscope-rs/commit/c347e1775487456b996b87ce4672a20d9b9f4541))
|
||||
* **deps:** update rust crate env_logger to v0.11.10 ([#490](https://github.com/grafana/pyroscope-rs/issues/490)) ([24fb12d](https://github.com/grafana/pyroscope-rs/commit/24fb12d4c26c9cc167585f3fd93172015864a0e4))
|
||||
* **deps:** update rust crate libc to v0.2.183 ([#489](https://github.com/grafana/pyroscope-rs/issues/489)) ([b44bac8](https://github.com/grafana/pyroscope-rs/commit/b44bac8f4274882d793e9e9f92ab0ad9fc4692f0))
|
||||
* **deps:** update rust crate libc to v0.2.185 ([#514](https://github.com/grafana/pyroscope-rs/issues/514)) ([e72f5c6](https://github.com/grafana/pyroscope-rs/commit/e72f5c6e4944e7b2ac949d6dbd3201dc426704d5))
|
||||
* **deps:** update rust crate libflate to v2.3.0 ([#516](https://github.com/grafana/pyroscope-rs/issues/516)) ([da2835f](https://github.com/grafana/pyroscope-rs/commit/da2835f536771e089635e9fd8b97a860b09b9479))
|
||||
* **deps:** update rust crate rbspy to 0.43 ([#471](https://github.com/grafana/pyroscope-rs/issues/471)) ([1e524d3](https://github.com/grafana/pyroscope-rs/commit/1e524d38d84dfedc799515cb70ee434ef91a28e6))
|
||||
* **deps:** update rust crate rbspy to 0.44 ([#477](https://github.com/grafana/pyroscope-rs/issues/477)) ([c7cfe39](https://github.com/grafana/pyroscope-rs/commit/c7cfe39fecd8a0624df5c5d22b709e73747ddb4d))
|
||||
* **deps:** update rust crate uuid to v1.23.0 ([#481](https://github.com/grafana/pyroscope-rs/issues/481)) ([9c4b53a](https://github.com/grafana/pyroscope-rs/commit/9c4b53aef55b13dc6da99396119e33ec9dc02aa0))
|
||||
* **deps:** update rust crate uuid to v1.23.1 ([#517](https://github.com/grafana/pyroscope-rs/issues/517)) ([6a89238](https://github.com/grafana/pyroscope-rs/commit/6a89238e7127cb5f3fa0f480ba7447369a21fc12))
|
||||
* **deps:** update rust-lang/crates-io-auth-action action to v1.0.4 ([#479](https://github.com/grafana/pyroscope-rs/issues/479)) ([f4e08f3](https://github.com/grafana/pyroscope-rs/commit/f4e08f34aa2b14a1e417c005d37f75981940d675))
|
||||
* **deps:** update rust:trixie docker digest to 652612f ([#510](https://github.com/grafana/pyroscope-rs/issues/510)) ([7f76541](https://github.com/grafana/pyroscope-rs/commit/7f76541afc2a03cd783b0a0627b7a876f18c4918))
|
||||
* **deps:** update softprops/action-gh-release action to v2.5.2 ([#473](https://github.com/grafana/pyroscope-rs/issues/473)) ([1700be3](https://github.com/grafana/pyroscope-rs/commit/1700be3e34e7351a2f5f030187a2fac1ce1082e9))
|
||||
* **deps:** update softprops/action-gh-release action to v2.5.3 ([#474](https://github.com/grafana/pyroscope-rs/issues/474)) ([12e34c1](https://github.com/grafana/pyroscope-rs/commit/12e34c109bd0e408dc0401430f4da4b5245a09da))
|
||||
* **deps:** update softprops/action-gh-release action to v2.6.1 ([#475](https://github.com/grafana/pyroscope-rs/issues/475)) ([209c4c0](https://github.com/grafana/pyroscope-rs/commit/209c4c03ce08fe411046316dc333654fb64ab75b))
|
||||
* remove FFI code and kindasafe (moved to separate repos) ([#488](https://github.com/grafana/pyroscope-rs/issues/488)) ([35a4c6a](https://github.com/grafana/pyroscope-rs/commit/35a4c6aa0808f0392e39e752d95a91536068ab74))
|
||||
|
||||
|
||||
### Documentation
|
||||
|
||||
* fix broken Rust documentation and add doc-test CI ([#484](https://github.com/grafana/pyroscope-rs/issues/484)) ([98ae8a5](https://github.com/grafana/pyroscope-rs/commit/98ae8a595f18946f2edf27544ea6e2bb29724cbf))
|
||||
|
||||
|
||||
### Continuous Integration
|
||||
|
||||
* add musl/Alpine to Rust test matrix ([#495](https://github.com/grafana/pyroscope-rs/issues/495)) ([a4b7ad4](https://github.com/grafana/pyroscope-rs/commit/a4b7ad4c9e530576adbbf278b104d65c970d43b0))
|
||||
* add release-please configuration ([#491](https://github.com/grafana/pyroscope-rs/issues/491)) ([fbb2d8e](https://github.com/grafana/pyroscope-rs/commit/fbb2d8edc7cf400343650e77580464d32369d93d))
|
||||
|
||||
## v2.0.0
|
||||
|
||||
## Breaking Changes
|
||||
- `Backend::report()` now returns `ReportBatch` instead of `Vec<Report>` ([#437](https://github.com/grafana/pyroscope-rs/pull/437), [#447](https://github.com/grafana/pyroscope-rs/pull/447))
|
||||
- New `ReportData` enum replaces the plain `Vec<Report>` in `ReportBatch`, supporting both structured reports (`ReportData::Reports`) and pre-encoded pprof bytes (`ReportData::RawPprof`) ([#447](https://github.com/grafana/pyroscope-rs/pull/447))
|
||||
- Minimum supported Rust version bumped from 1.64 to 1.66
|
||||
|
||||
## New Features
|
||||
- **Jemalloc memory profiling backend** — new `backend-jemalloc` feature flag enables heap profiling via `jemalloc_pprof`. Use `pyroscope::backend::jemalloc::jemalloc_backend()` to get started ([#378](https://github.com/grafana/pyroscope-rs/pull/378))
|
||||
- **`ReportBatch` type** — backends now return a `ReportBatch` with a `profile_type` field (e.g. `"process_cpu"`, `"memory"`), enabling multi-profile support ([#437](https://github.com/grafana/pyroscope-rs/pull/437))
|
||||
|
||||
## Dependencies
|
||||
- Updated `pprof` (pprof-pyroscope-fork) to v0.1500.3 ([#407](https://github.com/grafana/pyroscope-rs/pull/407))
|
||||
- Updated `object` crate to 0.38 ([#430](https://github.com/grafana/pyroscope-rs/pull/430))
|
||||
- Disabled py-spy default features to exclude CLI dependencies ([#418](https://github.com/grafana/pyroscope-rs/pull/418))
|
||||
- Added `jemalloc_pprof` 0.8 and `tokio` 1 as workspace dependencies
|
||||
|
||||
## v1.0.0
|
||||
## Breaking Changes
|
||||
- Removed `auth_token` from Python and Ruby FFI bindings and related code
|
||||
- Removed `detect_subprocesses` from Python and Ruby configs
|
||||
- Config constructor now requires app/spy identity and sample rate inputs
|
||||
- Removed support for collapsed format
|
||||
- Removed global tags from ruleset
|
||||
|
||||
## New Features
|
||||
- Integrated pprof-rs backend into main crate behind optional `backend-pprof-rs` feature
|
||||
- Switched to push API (from `/ingest` to `/push`)
|
||||
- Generated push API protos
|
||||
- Added `ThreadId` type
|
||||
- Added `rustls-no-provider` TLS feature
|
||||
|
||||
## Bug Fixes / Improvements
|
||||
- Unified signal logic
|
||||
- Report cleanup functions
|
||||
- Optimized ruleset
|
||||
- Removed obscure thread id hash check
|
||||
- Ruby: inline thread_id crate; remove detect_subprocess
|
||||
- Dependency updates (reqwest 0.13, prost 0.14, thiserror 2.0, serde_json 1.0.115, uuid 1.20, libflate 2.1)
|
||||
|
||||
## v0.5.4
|
||||
## New Features
|
||||
- Add report transformation function which allows changing the report before
|
||||
sending it to the Pyroscope Server.
|
||||
- Add Gzip support.
|
||||
|
||||
## Bug Fixes
|
||||
- Use URL Builder. ([786c89b](https://github.com/pyroscope-io/pyroscope-rs/commit/786c89bb99839c45778410012a6da60267d395df))
|
||||
|
||||
## v0.5.3
|
||||
## New Features
|
||||
- Add BackendConfig to make reporting of pid, thread_id and thread_name
|
||||
optional.
|
||||
- Backends can add a suffix to the "application_name"
|
||||
|
||||
## Bug Fixes
|
||||
- **main**: fixed an obsecure bug when counting stacktraces ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/bdecaa13aeae3ce7d4c3d97f88bdd104ec35e7c5))
|
||||
|
||||
## v0.5.2
|
||||
## New features
|
||||
- Authentication Token support
|
||||
|
||||
## API Changes
|
||||
- use rust-tls instead of openssl
|
||||
|
||||
## v0.5.1
|
||||
## API Changes
|
||||
- Backend shutdown bug
|
||||
- Docs update
|
||||
|
||||
## v0.5.0
|
||||
## API Changes
|
||||
- New API for building, starting and stopping the profiling agent.
|
||||
- Backend supports reporting multiple threads.
|
||||
- Tagging within local thread-scope
|
||||
|
||||
## v0.4.0
|
||||
## API Changes
|
||||
- Backend now support passing a configuration struct.
|
||||
- TimerSignal enum
|
||||
- pprof-rs backend is split into a different package. It has to be imported manually.
|
||||
|
||||
## What's Changed
|
||||
* fix: avoid binding two unrelated vars to the same type by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/18
|
||||
* avoid almost all unwraps by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/14
|
||||
* use more features of `thiserror` by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/11
|
||||
* introduce LOG_TAGs, avoid repetitive prefixes by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/10
|
||||
* allow configurable accumulation_cycle by @drahnr in https://github.com/pyroscope-io/pyroscope-rs/pull/21
|
||||
* Add CI Targets by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/22
|
||||
* 0.4.0 release by @omarabid in https://github.com/pyroscope-io/pyroscope-rs/pull/23
|
||||
|
||||
## New Contributors
|
||||
* @drahnr made their first contribution in https://github.com/pyroscope-io/pyroscope-rs/pull/18
|
||||
|
||||
**Full Changelog**: https://github.com/pyroscope-io/pyroscope-rs/compare/0.3.1...lib-0.4.0
|
||||
|
||||
## v0.3.1
|
||||
Minor release with bug fixes.
|
||||
|
||||
## Bug Fixes
|
||||
- **session**: avoid breaking SessionManager if request fails ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/1254bcc9a3b0d76b7adbeb82ba21f33b875c0b39))
|
||||
- **typo**: variable name typo ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/0d8caffbe7855bec8158333dba2923cd07286a5f))
|
||||
- **pprof**: fix #12 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/936d3d99a2cc82812bc8251fd2fbf8152a87d4cb))
|
||||
- **copyright**: fix #13 ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/b8eaf13137810df932e1b70e33b3ad3e25b65ece))
|
||||
|
||||
## Code Refactoring
|
||||
- **option**: replace unwrap for various Options ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/3fd4e794d74523855c66f65c0b7fc8ff9dfe4604))
|
||||
- **time**: add get_time_range ([Abid Omar](https://github.com/pyroscope-io/pyroscope-rs/commit/a6d4dbcef494b2bfe8016a817201499937cf3528))
|
||||
|
||||
## v0.3.0
|
||||
First stable release
|
||||
|
||||
## v0.0.2-alpha
|
||||
Second beta release
|
||||
|
||||
## v0.0.1-alpha
|
||||
Initial beta release
|
||||
Vendored
-233
@@ -1,233 +0,0 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
rust-version = "1.66"
|
||||
name = "pyroscope"
|
||||
version = "2.1.1"
|
||||
authors = ["Abid Omar <contact@omarabid.com>"]
|
||||
build = "build.rs"
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = """
|
||||
Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications.
|
||||
"""
|
||||
homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/"
|
||||
documentation = "https://docs.rs/pyroscope"
|
||||
readme = "README.md"
|
||||
keywords = [
|
||||
"pyroscope",
|
||||
"profiler",
|
||||
"profiling",
|
||||
"pprof",
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/grafana/pyroscope-rs"
|
||||
|
||||
[features]
|
||||
backend-jemalloc = ["dep:jemalloc_pprof"]
|
||||
backend-pprof-rs = [
|
||||
"dep:backtrace",
|
||||
"dep:backtrace",
|
||||
"dep:once_cell",
|
||||
"dep:nix",
|
||||
"dep:spin",
|
||||
"dep:tempfile",
|
||||
"dep:findshlibs",
|
||||
"dep:smallvec",
|
||||
"dep:framehop",
|
||||
"dep:memmap2",
|
||||
"dep:object",
|
||||
"dep:aligned-vec",
|
||||
"dep:symbolic-demangle",
|
||||
]
|
||||
default = ["rustls-tls"]
|
||||
native-tls = ["reqwest/native-tls"]
|
||||
native-tls-vendored = ["reqwest/native-tls-vendored"]
|
||||
rustls-no-provider = ["reqwest/rustls-no-provider"]
|
||||
rustls-tls = ["reqwest/rustls"]
|
||||
|
||||
[lib]
|
||||
name = "pyroscope"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[example]]
|
||||
name = "jemalloc"
|
||||
path = "examples/jemalloc.rs"
|
||||
required-features = ["backend-jemalloc"]
|
||||
|
||||
[[test]]
|
||||
name = "agent"
|
||||
path = "tests/agent.rs"
|
||||
|
||||
[[test]]
|
||||
name = "pprof_backend"
|
||||
path = "tests/pprof_backend.rs"
|
||||
|
||||
[[test]]
|
||||
name = "session"
|
||||
path = "tests/session.rs"
|
||||
|
||||
[[test]]
|
||||
name = "timer"
|
||||
path = "tests/timer.rs"
|
||||
|
||||
[[test]]
|
||||
name = "timer-epoll"
|
||||
path = "tests/timer-epoll.rs"
|
||||
|
||||
[dependencies.aligned-vec]
|
||||
version = "0.6"
|
||||
optional = true
|
||||
|
||||
[dependencies.backtrace]
|
||||
version = "0.3"
|
||||
optional = true
|
||||
|
||||
[dependencies.findshlibs]
|
||||
version = "0.10"
|
||||
optional = true
|
||||
|
||||
[dependencies.framehop]
|
||||
version = "0.16"
|
||||
optional = true
|
||||
|
||||
[dependencies.jemalloc_pprof]
|
||||
version = "0.9"
|
||||
features = ["symbolize"]
|
||||
optional = true
|
||||
|
||||
[dependencies.libc]
|
||||
version = "^0.2.124"
|
||||
|
||||
[dependencies.libflate]
|
||||
version = "2.1.0"
|
||||
|
||||
[dependencies.log]
|
||||
version = "0.4"
|
||||
|
||||
[dependencies.memmap2]
|
||||
version = "0.9"
|
||||
optional = true
|
||||
|
||||
[dependencies.nix]
|
||||
version = "0.31"
|
||||
features = [
|
||||
"signal",
|
||||
"fs",
|
||||
]
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.object]
|
||||
version = "0.40"
|
||||
optional = true
|
||||
|
||||
[dependencies.once_cell]
|
||||
version = "1.9"
|
||||
optional = true
|
||||
|
||||
[dependencies.prost]
|
||||
version = "0.14"
|
||||
|
||||
[dependencies.reqwest]
|
||||
version = "0.13"
|
||||
features = [
|
||||
"blocking",
|
||||
"query",
|
||||
]
|
||||
default-features = false
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1.0.115"
|
||||
|
||||
[dependencies.smallvec]
|
||||
version = "1.7"
|
||||
optional = true
|
||||
|
||||
[dependencies.spin]
|
||||
version = "0.12"
|
||||
optional = true
|
||||
|
||||
[dependencies.symbolic-demangle]
|
||||
version = "13.1.0"
|
||||
features = ["rust"]
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.tempfile]
|
||||
version = "3.1"
|
||||
optional = true
|
||||
|
||||
[dependencies.thiserror]
|
||||
version = "2.0"
|
||||
|
||||
[dependencies.url]
|
||||
version = "2.2.2"
|
||||
|
||||
[dependencies.uuid]
|
||||
version = "1.20.0"
|
||||
features = ["v4"]
|
||||
|
||||
[dev-dependencies.assert_matches]
|
||||
version = "=1.5.0"
|
||||
|
||||
[dev-dependencies.claims]
|
||||
version = "=0.8.0"
|
||||
|
||||
[dev-dependencies.env_logger]
|
||||
version = "=0.11.10"
|
||||
|
||||
[dev-dependencies.tikv-jemallocator]
|
||||
version = "0.7"
|
||||
features = ["profiling"]
|
||||
|
||||
[profile.bench]
|
||||
opt-level = 3
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
debug = 0
|
||||
debug-assertions = false
|
||||
rpath = false
|
||||
|
||||
[profile.dev]
|
||||
opt-level = 0
|
||||
lto = false
|
||||
codegen-units = 4
|
||||
debug = 2
|
||||
debug-assertions = true
|
||||
rpath = false
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
debug = 0
|
||||
debug-assertions = false
|
||||
rpath = false
|
||||
strip = "symbols"
|
||||
|
||||
[profile.test]
|
||||
opt-level = 1
|
||||
lto = false
|
||||
codegen-units = 4
|
||||
debug = 2
|
||||
debug-assertions = true
|
||||
rpath = false
|
||||
|
||||
[[test]]
|
||||
name = "rustfs_zstd"
|
||||
path = "tests/rustfs_zstd.rs"
|
||||
required-features = ["backend-pprof-rs"]
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
[package]
|
||||
name = "pyroscope"
|
||||
description = """
|
||||
Pyroscope Profiler Agent for continuous profiling of Rust, Python and Ruby applications.
|
||||
"""
|
||||
keywords = ["pyroscope", "profiler", "profiling", "pprof"]
|
||||
authors = ["Abid Omar <contact@omarabid.com>"]
|
||||
version = "2.1.1"
|
||||
edition = "2021"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/"
|
||||
documentation = "https://docs.rs/pyroscope"
|
||||
repository = "https://github.com/grafana/pyroscope-rs"
|
||||
readme = "README.md"
|
||||
autobins = false
|
||||
autoexamples = true
|
||||
autotests = true
|
||||
autobenches = true
|
||||
rust-version = "1.66"
|
||||
|
||||
[dependencies]
|
||||
thiserror = "2.0"
|
||||
log = "0.4"
|
||||
reqwest = { version = "0.13", features = ["blocking", "query"], default-features = false }
|
||||
uuid = { version = "1.20.0", features = ["v4"] }
|
||||
url = "2.2.2"
|
||||
libflate = "2.1.0"
|
||||
libc = "^0.2.124"
|
||||
prost = "0.14"
|
||||
serde_json = "1.0.115"
|
||||
jemalloc_pprof = { version = "0.9", features = ["symbolize"], optional = true }
|
||||
|
||||
|
||||
# pprofrs dependencies
|
||||
backtrace = { version = "0.3", optional = true }
|
||||
once_cell = { version = "1.9", optional = true }
|
||||
nix = { version = "0.31", default-features = false, features = ["signal", "fs"], optional = true }
|
||||
spin = { version = "0.12", optional = true }
|
||||
tempfile = { version = "3.1", optional = true }
|
||||
findshlibs = { version = "0.10", optional = true }
|
||||
smallvec = { version = "1.7", optional = true }
|
||||
framehop = { version = "0.16", optional = true }
|
||||
memmap2 = { version = "0.9", optional = true }
|
||||
object = { version = "0.40", optional = true }
|
||||
aligned-vec = { version = "0.6", optional = true }
|
||||
symbolic-demangle = { version = "13.1.0", default-features = false, features = ["rust"], optional = true }
|
||||
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
assert_matches = "=1.5.0"
|
||||
claims = "=0.8.0"
|
||||
tikv-jemallocator = { version = "0.7", features = ["profiling"] }
|
||||
env_logger = "=0.11.10"
|
||||
|
||||
[[example]]
|
||||
name = "jemalloc"
|
||||
required-features = ["backend-jemalloc"]
|
||||
|
||||
[features]
|
||||
default = ["rustls-tls"]
|
||||
rustls-tls = ["reqwest/rustls"]
|
||||
native-tls = ["reqwest/native-tls"]
|
||||
native-tls-vendored = ["reqwest/native-tls-vendored"]
|
||||
rustls-no-provider = ["reqwest/rustls-no-provider"]
|
||||
backend-pprof-rs = [
|
||||
"dep:backtrace",
|
||||
"dep:backtrace",
|
||||
"dep:once_cell",
|
||||
"dep:nix",
|
||||
"dep:spin",
|
||||
"dep:tempfile",
|
||||
"dep:findshlibs",
|
||||
"dep:smallvec",
|
||||
"dep:framehop",
|
||||
"dep:memmap2",
|
||||
"dep:object",
|
||||
"dep:aligned-vec",
|
||||
"dep:symbolic-demangle",
|
||||
]
|
||||
backend-jemalloc = ["dep:jemalloc_pprof"]
|
||||
|
||||
[profile.dev]
|
||||
opt-level = 0
|
||||
debug = true
|
||||
rpath = false
|
||||
lto = false
|
||||
debug-assertions = true
|
||||
codegen-units = 4
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
debug = false
|
||||
strip = "symbols"
|
||||
rpath = false
|
||||
lto = true
|
||||
debug-assertions = false
|
||||
codegen-units = 1
|
||||
|
||||
[profile.test]
|
||||
opt-level = 1
|
||||
debug = true
|
||||
rpath = false
|
||||
lto = false
|
||||
debug-assertions = true
|
||||
codegen-units = 4
|
||||
|
||||
[profile.bench]
|
||||
opt-level = 3
|
||||
debug = false
|
||||
rpath = false
|
||||
lto = true
|
||||
debug-assertions = false
|
||||
codegen-units = 1
|
||||
|
||||
[[test]]
|
||||
name = "rustfs_zstd"
|
||||
path = "tests/rustfs_zstd.rs"
|
||||
required-features = ["backend-pprof-rs"]
|
||||
Vendored
-202
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2022 Pyroscope
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Vendored
-22
@@ -1,22 +0,0 @@
|
||||
## Pyroscope Profiler
|
||||
|
||||
**Pyroscope Profiler for Rust. Profile your Rust applications.**
|
||||
|
||||
[](LICENSE)
|
||||
[](https://crates.io/crates/pyroscope)
|
||||
|
||||
|
||||
### Major Contributors
|
||||
|
||||
We'd like to give a big thank you to the following contributors who have made significant contributions to this project:
|
||||
|
||||
* [Abid Omar](https://github.com/omarabid)
|
||||
* [Anatoly Korniltsev](https://github.com/korniltsev)
|
||||
* [Bernhard Schuster](https://github.com/drahnr)
|
||||
|
||||
|
||||
### License
|
||||
|
||||
Pyroscope is distributed under the Apache License (Version 2.0).
|
||||
|
||||
See [LICENSE](LICENSE) for details.
|
||||
Vendored
-9
@@ -1,9 +0,0 @@
|
||||
# Temporary Pyroscope dependency patch
|
||||
|
||||
This is the published [pyroscope 2.1.1](https://crates.io/crates/pyroscope/2.1.1) source, from upstream commit `bc958d9e9d934ab97bc3c6887ed62c5880467d6b`, with its original license retained. Repository automation is omitted and documentation trailing whitespace is normalized. Copied comments and documentation also have spelling corrections required by the repository typo check.
|
||||
|
||||
The only runtime dependency change is the `object` dependency requirement, from `0.39` to `0.40`, in both manifests. Runtime behavior, features, build script, examples, and original tests are unchanged; source edits beyond the dependency requirement only correct comments. `tests/rustfs_zstd.rs` adds bounded valid and oversized-block regression coverage through `object`'s decompression API, registered in both manifests. `object 0.39` requires vulnerable `ruzstd 0.8`; `object 0.40` permits the patched `ruzstd 0.9.1` required by [RUSTSEC-2026-0344](https://rustsec.org/advisories/RUSTSEC-2026-0344). Keep the patched version in the workspace lockfile and retain the existing Cargo Deny gate.
|
||||
|
||||
Remove this directory, the workspace exclusion, and the `[patch.crates-io]` entry once an upstream Pyroscope release accepts `object 0.40` or later. Track validation and removal in [rustfs/backlog#2825](https://github.com/rustfs/backlog/issues/2825).
|
||||
|
||||
Run the retained upstream unit tests and the RustFS regressions with `cargo test --manifest-path third-party/pyroscope/Cargo.toml --features backend-pprof-rs --lib --test rustfs_zstd`.
|
||||
Vendored
-19
@@ -1,19 +0,0 @@
|
||||
use std::env;
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
let rustc = env::var("RUSTC").unwrap();
|
||||
|
||||
let output = Command::new(rustc)
|
||||
.arg("--version")
|
||||
.output()
|
||||
.expect("Failed to run rustc");
|
||||
|
||||
let version_string =
|
||||
String::from_utf8(output.stdout).expect("rustc --version stdout is not utf8");
|
||||
|
||||
println!(
|
||||
"cargo:rustc-env=PYROSCOPE__RUSTC_VERSION={}",
|
||||
version_string.trim()
|
||||
);
|
||||
}
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
//! Jemalloc memory profiling example.
|
||||
//!
|
||||
//! This example demonstrates how to use the jemalloc backend for memory profiling.
|
||||
//! It requires jemalloc to be configured as the global allocator with profiling enabled.
|
||||
//!
|
||||
//! # Requirements
|
||||
//!
|
||||
//! Add these dependencies to your Cargo.toml:
|
||||
//!
|
||||
//! ```toml
|
||||
//! [dependencies]
|
||||
//! pyroscope = { version = "2.0.0", features = ["backend-jemalloc"] }
|
||||
//! tikv-jemallocator = { version = "0.6", features = ["profiling"] }
|
||||
//! ```
|
||||
//!
|
||||
//! # Running
|
||||
//!
|
||||
//! ```sh
|
||||
//! # Enable jemalloc profiling via environment variable
|
||||
//! _RJEM_MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:19 \
|
||||
//! cargo run --example jemalloc --features backend-jemalloc
|
||||
//! ```
|
||||
|
||||
use pyroscope::backend::jemalloc::jemalloc_backend;
|
||||
use pyroscope::pyroscope::PyroscopeAgentBuilder;
|
||||
use std::time::Duration;
|
||||
|
||||
// Configure jemalloc as the global allocator.
|
||||
// Profiling must also be enabled at runtime via MALLOC_CONF or _RJEM_MALLOC_CONF.
|
||||
#[global_allocator]
|
||||
static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
env_logger::init();
|
||||
|
||||
let agent = PyroscopeAgentBuilder::new(
|
||||
"http://localhost:4040",
|
||||
"example.jemalloc",
|
||||
100,
|
||||
"pyroscope-rs",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
jemalloc_backend(),
|
||||
)
|
||||
.tags(vec![("env", "dev")])
|
||||
.build()?;
|
||||
|
||||
let agent_running = agent.start()?;
|
||||
|
||||
// Simulate heavy allocations for 30 seconds.
|
||||
let start = std::time::Instant::now();
|
||||
let mut iteration = 0u64;
|
||||
while start.elapsed() < Duration::from_secs(30) {
|
||||
// Allocate vectors of varying sizes (1KB to 1MB).
|
||||
for i in 0..100 {
|
||||
let size = 1024 * (1 + (iteration as usize + i) % 1024);
|
||||
let v: Vec<u8> = vec![0u8; size];
|
||||
std::hint::black_box(&v);
|
||||
}
|
||||
iteration += 1;
|
||||
}
|
||||
eprintln!("Completed {iteration} iterations");
|
||||
|
||||
let agent_ready = agent_running.stop()?;
|
||||
agent_ready.shutdown();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
-134
@@ -1,134 +0,0 @@
|
||||
#![allow(clippy::module_inception)]
|
||||
|
||||
use crate::error::{PyroscopeError, Result};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use super::{ReportBatch, ThreadTag};
|
||||
|
||||
/// Backend Config
|
||||
#[derive(Debug, Copy, Clone, Default)]
|
||||
pub struct BackendConfig {
|
||||
pub report_thread_id: bool,
|
||||
pub report_thread_name: bool,
|
||||
pub report_pid: bool,
|
||||
}
|
||||
|
||||
/// Backend Trait
|
||||
pub trait Backend: Send {
|
||||
/// Initialize the backend.
|
||||
fn initialize(&mut self) -> Result<()>;
|
||||
/// Drop the backend.
|
||||
fn shutdown(self: Box<Self>) -> Result<()>;
|
||||
/// Generate profiling report
|
||||
fn report(&mut self) -> Result<ReportBatch>;
|
||||
fn add_tag(&self, tag: ThreadTag) -> Result<()>;
|
||||
fn remove_tag(&self, tag: ThreadTag) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Marker struct for Empty BackendImpl
|
||||
#[derive(Debug)]
|
||||
pub struct BackendBare;
|
||||
|
||||
/// Marker struct for Uninitialized Backend
|
||||
#[derive(Debug)]
|
||||
pub struct BackendUninitialized;
|
||||
|
||||
/// Marker struct for Initialized Backend
|
||||
#[derive(Debug)]
|
||||
pub struct BackendReady;
|
||||
|
||||
/// Backend State Trait
|
||||
pub trait BackendState {}
|
||||
impl BackendState for BackendBare {}
|
||||
impl BackendState for BackendUninitialized {}
|
||||
impl BackendState for BackendReady {}
|
||||
|
||||
/// Backend Accessibility Trait
|
||||
pub trait BackendAccessible: BackendState {}
|
||||
impl BackendAccessible for BackendUninitialized {}
|
||||
impl BackendAccessible for BackendReady {}
|
||||
|
||||
/// Precursor Backend Implementation
|
||||
/// This struct is used to implement the Backend trait. It serves two purposes:
|
||||
/// 1. It enforces state transitions using the Type System.
|
||||
/// 2. It manages the lifetime of the backend through an Arc<Mutex<T>>.
|
||||
pub struct BackendImpl<S: BackendState + ?Sized> {
|
||||
/// Backend
|
||||
pub backend: Arc<Mutex<Option<Box<dyn Backend>>>>,
|
||||
|
||||
/// Backend State
|
||||
_state: std::marker::PhantomData<S>,
|
||||
}
|
||||
|
||||
impl BackendImpl<BackendBare> {
|
||||
/// Create a new BackendImpl instance
|
||||
pub fn new(backend_box: Box<dyn Backend>) -> BackendImpl<BackendUninitialized> {
|
||||
BackendImpl {
|
||||
backend: Arc::new(Mutex::new(Some(backend_box))),
|
||||
_state: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendImpl<BackendUninitialized> {
|
||||
/// Initialize the backend
|
||||
pub fn initialize(self) -> Result<BackendImpl<BackendReady>> {
|
||||
let backend = self.backend.clone();
|
||||
|
||||
// Initialize the backend
|
||||
backend
|
||||
.lock()?
|
||||
.as_mut()
|
||||
.ok_or(PyroscopeError::BackendImpl)?
|
||||
.initialize()?;
|
||||
|
||||
// Transition to BackendReady
|
||||
Ok(BackendImpl {
|
||||
backend,
|
||||
_state: std::marker::PhantomData,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BackendAccessible> BackendImpl<S> {
|
||||
pub fn add_tag(&self, tag: ThreadTag) -> Result<()> {
|
||||
self.backend
|
||||
.lock()?
|
||||
.as_ref()
|
||||
.ok_or(PyroscopeError::BackendImpl)?
|
||||
.add_tag(tag)
|
||||
}
|
||||
|
||||
pub fn remove_tag(&self, rule: ThreadTag) -> Result<()> {
|
||||
self.backend
|
||||
.lock()?
|
||||
.as_ref()
|
||||
.ok_or(PyroscopeError::BackendImpl)?
|
||||
.remove_tag(rule)
|
||||
}
|
||||
}
|
||||
|
||||
impl BackendImpl<BackendReady> {
|
||||
/// Shutdown the backend and destroy BackendImpl
|
||||
pub fn shutdown(self) -> Result<()> {
|
||||
self.backend
|
||||
.lock()?
|
||||
.take()
|
||||
.ok_or(PyroscopeError::BackendImpl)?
|
||||
.shutdown()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate profiling report
|
||||
pub fn report(&mut self) -> Result<ReportBatch> {
|
||||
self.backend
|
||||
.lock()?
|
||||
.as_mut()
|
||||
.ok_or(PyroscopeError::BackendImpl)?
|
||||
.report()
|
||||
}
|
||||
}
|
||||
-87
@@ -1,87 +0,0 @@
|
||||
use crate::backend::{
|
||||
Backend, BackendImpl, BackendUninitialized, ReportBatch, ReportData, ThreadTag,
|
||||
};
|
||||
use crate::error::{PyroscopeError, Result};
|
||||
|
||||
const LOG_TAG: &str = "Pyroscope::Jemalloc";
|
||||
|
||||
/// Create a jemalloc memory profiling backend.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// use pyroscope::pyroscope::PyroscopeAgentBuilder;
|
||||
/// use pyroscope::backend::jemalloc::jemalloc_backend;
|
||||
///
|
||||
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let agent = PyroscopeAgentBuilder::new(
|
||||
/// "http://localhost:4040", "my-app", 100,
|
||||
/// "pyroscope-rs", env!("CARGO_PKG_VERSION"),
|
||||
/// jemalloc_backend(),
|
||||
/// )
|
||||
/// .build()?;
|
||||
/// agent.start()?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn jemalloc_backend() -> BackendImpl<BackendUninitialized> {
|
||||
BackendImpl::new(Box::new(Jemalloc))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Jemalloc;
|
||||
|
||||
impl Backend for Jemalloc {
|
||||
fn initialize(&mut self) -> Result<()> {
|
||||
let prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().ok_or_else(|| {
|
||||
PyroscopeError::new(
|
||||
"jemalloc: PROF_CTL not available. Ensure jemalloc is configured with prof:true",
|
||||
)
|
||||
})?;
|
||||
let guard = prof_ctl.try_lock().map_err(|_| {
|
||||
PyroscopeError::new(
|
||||
"jemalloc: failed to acquire PROF_CTL lock during initialization. \
|
||||
This is unexpected at startup; ensure no other code holds the lock.",
|
||||
)
|
||||
})?;
|
||||
if !guard.activated() {
|
||||
return Err(PyroscopeError::new(
|
||||
"jemalloc: profiling is not activated. Ensure malloc_conf includes prof:true,prof_active:true",
|
||||
));
|
||||
}
|
||||
|
||||
log::info!(target: LOG_TAG, "Jemalloc profiling backend initialized");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shutdown(self: Box<Self>) -> Result<()> {
|
||||
log::trace!(target: LOG_TAG, "Shutting down jemalloc backend");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report(&mut self) -> Result<ReportBatch> {
|
||||
let prof_ctl = jemalloc_pprof::PROF_CTL
|
||||
.as_ref()
|
||||
.ok_or_else(|| PyroscopeError::new("jemalloc: PROF_CTL not available"))?;
|
||||
let mut guard = prof_ctl.try_lock().map_err(|_| {
|
||||
PyroscopeError::new("jemalloc: failed to acquire PROF_CTL lock for report")
|
||||
})?;
|
||||
let pprof_data = guard
|
||||
.dump_pprof()
|
||||
.map_err(|e| PyroscopeError::new(&format!("jemalloc: dump_pprof failed: {e}")))?;
|
||||
|
||||
Ok(ReportBatch {
|
||||
profile_type: "memory".into(),
|
||||
data: ReportData::RawPprof(pprof_data),
|
||||
})
|
||||
}
|
||||
|
||||
fn add_tag(&self, _tag: ThreadTag) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_tag(&self, _tag: ThreadTag) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
pub mod backend;
|
||||
#[cfg(feature = "backend-jemalloc")]
|
||||
pub mod jemalloc;
|
||||
#[cfg(all(
|
||||
feature = "backend-pprof-rs",
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
),
|
||||
all(
|
||||
target_os = "macos",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
)
|
||||
)
|
||||
))]
|
||||
pub mod pprof;
|
||||
#[cfg(all(
|
||||
feature = "backend-pprof-rs",
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
),
|
||||
all(
|
||||
target_os = "macos",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
)
|
||||
)
|
||||
))]
|
||||
mod pprofrs;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "backend-pprof-rs",
|
||||
any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
),
|
||||
all(
|
||||
target_os = "macos",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
)
|
||||
)
|
||||
))]
|
||||
pub use pprof::*;
|
||||
|
||||
#[cfg(all(
|
||||
feature = "backend-pprof-rs",
|
||||
not(any(
|
||||
all(
|
||||
target_os = "linux",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
),
|
||||
all(
|
||||
target_os = "macos",
|
||||
any(target_arch = "x86_64", target_arch = "aarch64")
|
||||
)
|
||||
))
|
||||
))]
|
||||
compile_error!("feature `backend-pprof-rs` is only supported on Linux/macOS x86_64/aarch64");
|
||||
|
||||
pub mod ruleset;
|
||||
pub mod tests;
|
||||
pub mod types;
|
||||
|
||||
pub use backend::*;
|
||||
#[cfg(feature = "backend-jemalloc")]
|
||||
pub use jemalloc::*;
|
||||
pub use ruleset::*;
|
||||
pub use types::*;
|
||||
-240
@@ -1,240 +0,0 @@
|
||||
use crate::backend::pprofrs::{ProfilerGuard, ProfilerGuardBuilder};
|
||||
use crate::backend::{
|
||||
Backend, BackendConfig, BackendImpl, BackendUninitialized, Report, ReportBatch, ReportData,
|
||||
StackBuffer, StackFrame, StackTrace, ThreadTag, ThreadTagsSet,
|
||||
};
|
||||
use crate::error::{PyroscopeError, Result};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
ffi::OsStr,
|
||||
ops::Deref,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
const LOG_TAG: &str = "Pyroscope::Pprofrs";
|
||||
|
||||
pub fn pprof_backend(
|
||||
config: PprofConfig,
|
||||
backend_config: BackendConfig,
|
||||
) -> BackendImpl<BackendUninitialized> {
|
||||
BackendImpl::new(Box::new(Pprof::new(config, backend_config)))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PprofConfig {
|
||||
pub sample_rate: u32,
|
||||
}
|
||||
|
||||
impl Default for PprofConfig {
|
||||
fn default() -> Self {
|
||||
PprofConfig { sample_rate: 100 }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Pprof<'a> {
|
||||
buffer: Arc<Mutex<StackBuffer>>,
|
||||
config: PprofConfig,
|
||||
backend_config: BackendConfig,
|
||||
guard: Arc<Mutex<Option<ProfilerGuard<'a>>>>,
|
||||
ruleset: ThreadTagsSet,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Pprof<'_> {
|
||||
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
|
||||
write!(fmt, "Pprof Backend")
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Pprof<'a> {
|
||||
pub fn new(config: PprofConfig, backend_config: BackendConfig) -> Self {
|
||||
Pprof {
|
||||
buffer: Arc::new(Mutex::new(StackBuffer::default())),
|
||||
config,
|
||||
backend_config,
|
||||
guard: Arc::new(Mutex::new(None)),
|
||||
ruleset: ThreadTagsSet::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Backend for Pprof<'_> {
|
||||
fn shutdown(self: Box<Self>) -> Result<()> {
|
||||
log::trace!(target: LOG_TAG, "Shutting down sampler thread");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn initialize(&mut self) -> Result<()> {
|
||||
let profiler = ProfilerGuardBuilder::default()
|
||||
.frequency(self.config.sample_rate as i32)
|
||||
.build()
|
||||
.map_err(|e| PyroscopeError::new(e.to_string().as_str()))?;
|
||||
|
||||
*self.guard.lock()? = Some(profiler);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report(&mut self) -> Result<ReportBatch> {
|
||||
self.dump_report()?;
|
||||
|
||||
let buffer = self.buffer.clone();
|
||||
|
||||
let report: StackBuffer = buffer.lock()?.deref().to_owned();
|
||||
|
||||
let reports: Vec<Report> = report.into();
|
||||
|
||||
buffer.lock()?.clear();
|
||||
|
||||
Ok(ReportBatch {
|
||||
profile_type: "process_cpu".into(),
|
||||
data: ReportData::Reports(reports),
|
||||
})
|
||||
}
|
||||
|
||||
fn add_tag(&self, tag: ThreadTag) -> Result<()> {
|
||||
if self.guard.lock()?.as_ref().is_some() {
|
||||
self.dump_report()?;
|
||||
}
|
||||
|
||||
self.ruleset.add(tag)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_tag(&self, tag: ThreadTag) -> Result<()> {
|
||||
if self.guard.lock()?.as_ref().is_some() {
|
||||
self.dump_report()?;
|
||||
}
|
||||
|
||||
self.ruleset.remove(tag)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Pprof<'_> {
|
||||
/// Workaround for pprof-rs to interrupt the profiler.
|
||||
pub fn dump_report(&self) -> Result<()> {
|
||||
let report = self
|
||||
.guard
|
||||
.lock()?
|
||||
.as_ref()
|
||||
.ok_or_else(|| PyroscopeError::new("pprof-rs: ProfilerGuard report error"))?
|
||||
.report()
|
||||
.build_and_clear(true)
|
||||
.map_err(|e| PyroscopeError::new(e.to_string().as_str()))?;
|
||||
|
||||
let stack_buffer = Into::<StackBuffer>::into(Into::<StackBufferWrapper>::into((
|
||||
report,
|
||||
&self.backend_config,
|
||||
)));
|
||||
|
||||
let data: HashMap<StackTrace, usize> = stack_buffer
|
||||
.data
|
||||
.iter()
|
||||
.map(|(stacktrace, ss)| {
|
||||
let stacktrace = stacktrace.to_owned().add_tag_rules(&self.ruleset);
|
||||
(stacktrace, ss.to_owned())
|
||||
})
|
||||
.collect();
|
||||
|
||||
let buffer = self.buffer.clone();
|
||||
|
||||
for (stacktrace, count) in data {
|
||||
buffer.lock()?.record_with_count(stacktrace, count)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct StackBufferWrapper(StackBuffer);
|
||||
|
||||
impl From<StackBufferWrapper> for StackBuffer {
|
||||
fn from(stackbuffer: StackBufferWrapper) -> Self {
|
||||
stackbuffer.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<(crate::backend::pprofrs::Report, &BackendConfig)> for StackBufferWrapper {
|
||||
fn from(arg: (crate::backend::pprofrs::Report, &BackendConfig)) -> Self {
|
||||
let (report, config) = arg;
|
||||
let buffer_data: HashMap<StackTrace, usize> = report
|
||||
.data
|
||||
.iter()
|
||||
.map(|(key, value)| {
|
||||
(
|
||||
Into::<StackTraceWrapper>::into((key.to_owned(), config)).into(),
|
||||
value.to_owned() as usize,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
StackBufferWrapper(StackBuffer::new(buffer_data))
|
||||
}
|
||||
}
|
||||
|
||||
struct StackTraceWrapper(StackTrace);
|
||||
|
||||
impl From<StackTraceWrapper> for StackTrace {
|
||||
fn from(stack_trace: StackTraceWrapper) -> Self {
|
||||
stack_trace.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<(crate::backend::pprofrs::Frames, &BackendConfig)> for StackTraceWrapper {
|
||||
fn from(arg: (crate::backend::pprofrs::Frames, &BackendConfig)) -> Self {
|
||||
let (frames, config) = arg;
|
||||
let thread_id = frames.thread_id as libc::pthread_t;
|
||||
StackTraceWrapper(StackTrace::new(
|
||||
config,
|
||||
None,
|
||||
Some(thread_id.into()),
|
||||
Some(frames.thread_name),
|
||||
frames
|
||||
.frames
|
||||
.concat()
|
||||
.iter()
|
||||
.map(|frame| Into::<StackFrameWrapper>::into(frame.to_owned()).into())
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct StackFrameWrapper(StackFrame);
|
||||
|
||||
impl From<StackFrameWrapper> for StackFrame {
|
||||
fn from(stack_frame: StackFrameWrapper) -> Self {
|
||||
stack_frame.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::backend::pprofrs::Symbol> for StackFrameWrapper {
|
||||
fn from(symbol: crate::backend::pprofrs::Symbol) -> Self {
|
||||
StackFrameWrapper(StackFrame::new(
|
||||
None,
|
||||
Some(symbol.name()),
|
||||
Some(
|
||||
symbol
|
||||
.filename
|
||||
.clone()
|
||||
.unwrap_or_default()
|
||||
.file_name()
|
||||
.unwrap_or_else(|| OsStr::new(""))
|
||||
.to_str()
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
),
|
||||
Some(
|
||||
symbol
|
||||
.filename
|
||||
.unwrap_or_default()
|
||||
.to_str()
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
),
|
||||
None,
|
||||
symbol.lineno,
|
||||
))
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user