mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 10:43:15 +00:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65035481f6 | |||
| b2ff43eb1a | |||
| 355c8d2e22 | |||
| 60eb139db9 | |||
| 9ef059c908 | |||
| 51497cb533 | |||
| c7a29ec0a7 | |||
| deb0edb7cc | |||
| a08de9229b | |||
| abffa5cf1b | |||
| b825c54850 | |||
| de9145e87a | |||
| 84bd76a3ce | |||
| 9a2d06b370 | |||
| 00de43528c | |||
| daecb93139 | |||
| 35a30cd614 | |||
| 360bceafce | |||
| 7cb91a0190 | |||
| beb6e1383e | |||
| 59b7d13095 | |||
| e0b87b0e7e | |||
| 21c2fb42bb | |||
| 984c705713 | |||
| 23b17c2d5a | |||
| 89e2513205 | |||
| d091554ffe | |||
| c04ee41cf0 |
@@ -15,28 +15,35 @@
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2.
|
||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
||||
#
|
||||
# Trigger:
|
||||
# - release published: automatically package when a GitHub release is published
|
||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||
# - workflow_run: automatically package after "Build and Release" completes
|
||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
||||
# to the GitHub release before packaging starts)
|
||||
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
||||
#
|
||||
# Flow:
|
||||
# 1. Find the Build workflow run for the release tag
|
||||
# 1. Resolve the triggering Build workflow run for the release tag
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64
|
||||
# 4. Build RPM packages for x86_64 and aarch64
|
||||
# 5. Upload all packages to Cloudflare R2
|
||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# contents: write is required to upload packages to the GitHub release
|
||||
contents: write
|
||||
actions: read
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [ published ]
|
||||
# Follows the same pattern as docker.yml: run after the release build
|
||||
# workflow completes, so packaging is triggered only by release tags
|
||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -49,13 +56,26 @@ on:
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
||||
# Tag pushes arrive as event == push with head_branch != main (a
|
||||
# non-main push head_branch is the release tag name). Manual dispatch
|
||||
# stays available as a fallback for backfills and re-runs.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -75,8 +95,8 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
# Determine tag
|
||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||
TAG="${{ github.event.release.tag_name }}"
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
TAG="${HEAD_BRANCH}"
|
||||
elif [[ -n "$INPUT_TAG" ]]; then
|
||||
TAG="$INPUT_TAG"
|
||||
else
|
||||
@@ -93,6 +113,11 @@ jobs:
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Use the Build and Release run that triggered this workflow
|
||||
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
||||
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
# Find the build run that produced this tag
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
@@ -456,6 +481,54 @@ jobs:
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TAG="${{ needs.resolve.outputs.tag }}"
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||
|
||||
# Upload the packages, then refresh the release checksums so the new
|
||||
# assets are covered, matching the binary release flow.
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
||||
gh release upload "$TAG" "$f" --clobber
|
||||
fi
|
||||
done
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
touch "$checksum_file"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
# Remove any stale entry, then append the fresh digest
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
mv "${checksum_file}.tmp" "$checksum_file"
|
||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
||||
gh release upload "$TAG" "$checksum_file" --clobber
|
||||
done
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
Generated
+158
-150
File diff suppressed because it is too large
Load Diff
+55
-55
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.3" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.3" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.3" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.3" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.3" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.3" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.3" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.3" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.3" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.3" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.3" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.3" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.3" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.3" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.3" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.3" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.3" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.3", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.3" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.3" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.3" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.3" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.3" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.3" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.3" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.3" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.3" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.3" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.3" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.3" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.3" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.3" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.3" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -228,9 +228,9 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
@@ -284,8 +284,8 @@ rayon = "1.12.0"
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
redis = { version = "1.5.0" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.6.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
@@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.24.0" }
|
||||
uuid = { version = "1.24.1" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.6" }
|
||||
russh = { version = "0.62.7" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -350,7 +350,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -40,6 +40,7 @@ mak = "mak"
|
||||
gae = "gae"
|
||||
GAE = "GAE"
|
||||
thr = "thr"
|
||||
mis = "mis"
|
||||
# s3-tests original test names (cannot be changed)
|
||||
nonexisted = "nonexisted"
|
||||
consts = "consts"
|
||||
|
||||
@@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -287,6 +287,9 @@ pub enum HealRequestSource {
|
||||
Scanner,
|
||||
AutoHeal,
|
||||
ReadRepair,
|
||||
/// Mission Repair Feed: intents delivered by error paths and replayed
|
||||
/// from the durable MRF journal.
|
||||
Mrf,
|
||||
}
|
||||
|
||||
impl HealRequestSource {
|
||||
@@ -297,6 +300,7 @@ impl HealRequestSource {
|
||||
Self::Scanner => "scanner",
|
||||
Self::AutoHeal => "auto_heal",
|
||||
Self::ReadRepair => "read_repair",
|
||||
Self::Mrf => "mrf",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,8 +17,10 @@ pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
pub mod mrf_channel;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
pub mod trace_bus;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Mission Repair Feed (MRF) intent channel.
|
||||
//!
|
||||
//! Producers on error paths (read decode failure, scanner metadata
|
||||
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
||||
//! heal crate through a global bounded channel. Delivery is strictly
|
||||
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
||||
//! (counting it) when the channel is full or uninitialized — losing one heal
|
||||
//! hint is always preferred over stalling an IO path. Durable replay of
|
||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
||||
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
||||
/// dropping (and counting) intents, never by blocking the producer.
|
||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
||||
|
||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
||||
/// PartialWrite -> Normal).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfKind {
|
||||
/// Erasure decode failed while serving a read (read path).
|
||||
DecodeFailure,
|
||||
/// Scanner classified object metadata as corrupt.
|
||||
MetadataCorruption,
|
||||
/// A write left the object with fewer committed shards than the set size.
|
||||
PartialWrite,
|
||||
}
|
||||
|
||||
impl MrfKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
MrfKind::DecodeFailure => "decode-failure",
|
||||
MrfKind::MetadataCorruption => "metadata-corruption",
|
||||
MrfKind::PartialWrite => "partial-write",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
||||
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
||||
/// intent never re-allocates the strings.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MrfIntent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
/// Version the intent targets, as raw UUID bytes.
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
pub kind: MrfKind,
|
||||
pub enqueued_at_ms: u64,
|
||||
/// Times this intent has already been offered to the heal manager.
|
||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
||||
pub attempts: u8,
|
||||
}
|
||||
|
||||
/// Consumer-side retry ceiling before an intent is given up on.
|
||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
||||
|
||||
impl MrfIntent {
|
||||
/// Rough in-memory footprint used by the queue's byte budget.
|
||||
pub fn estimated_bytes(&self) -> usize {
|
||||
// Struct + strings + version bytes; buckets and objects are usually
|
||||
// far below this bound, so rounding up keeps the budget conservative.
|
||||
64 + self.bucket.len() + self.object.len()
|
||||
}
|
||||
}
|
||||
|
||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
||||
|
||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
||||
/// this before touching the channel so the disabled path stays allocation- and
|
||||
/// sync-free.
|
||||
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
||||
|
||||
/// Override delivery (used at heal-runtime startup from configuration).
|
||||
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
||||
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether producers currently deliver intents.
|
||||
pub fn mrf_delivery_enabled() -> bool {
|
||||
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Create the global MRF channel and return the consumer half. Fails if the
|
||||
/// channel is already initialized (the heal runtime is a singleton).
|
||||
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
||||
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
||||
GLOBAL_MRF_SENDER
|
||||
.set(sender)
|
||||
.map_err(|_| "MRF channel sender already initialized")?;
|
||||
Ok(receiver)
|
||||
}
|
||||
|
||||
/// Best-effort, non-blocking intent delivery from an error path.
|
||||
///
|
||||
/// Returns `true` when the intent was accepted into the channel. `false`
|
||||
/// means the intent was dropped (feature disabled, channel not yet
|
||||
/// initialized, or channel full) — callers must not retry or await; the
|
||||
/// existing read-repair / scanner heal paths remain the safety net.
|
||||
///
|
||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
||||
if !mrf_delivery_enabled() {
|
||||
return false;
|
||||
}
|
||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
||||
return false;
|
||||
};
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
||||
kind,
|
||||
enqueued_at_ms: unix_now_ms(),
|
||||
attempts: 0,
|
||||
};
|
||||
sender.try_send(intent).is_ok()
|
||||
}
|
||||
|
||||
fn unix_now_ms() -> u64 {
|
||||
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
||||
// failure would be a bug rather than something to handle here.
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn intents_estimate_is_conservative() {
|
||||
let intent = MrfIntent {
|
||||
bucket: Arc::from("bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: Some([0u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
};
|
||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
||||
|
||||
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
||||
let intent = receiver.recv().await.expect("intent should arrive");
|
||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(intent.bucket.as_ref(), "b");
|
||||
|
||||
// Disable delivery: producers become no-ops.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
|
||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
||||
// never blocking.
|
||||
let mut accepted = 0;
|
||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
||||
accepted += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_send_without_channel_is_false() {
|
||||
// This test may run after the tokio test above in the same process;
|
||||
// the singleton semantics make a clean "uninitialized" case hard, so
|
||||
// assert the flag-off behavior only.
|
||||
set_mrf_delivery_enabled(false);
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
||||
|
||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceKind {
|
||||
Heal,
|
||||
Scanner,
|
||||
}
|
||||
|
||||
impl TraceKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Heal => "heal",
|
||||
Self::Scanner => "scanner",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TraceFunc {
|
||||
HealTask,
|
||||
HealBucket,
|
||||
HealObject,
|
||||
HealCheckAbandonedParts,
|
||||
HealErasureSetPage,
|
||||
ScannerFolder,
|
||||
ScannerIlmAction,
|
||||
ScannerHealCandidate,
|
||||
Dropped,
|
||||
}
|
||||
|
||||
impl TraceFunc {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HealTask => "heal.Task",
|
||||
Self::HealBucket => "heal.Bucket",
|
||||
Self::HealObject => "heal.Object",
|
||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
||||
Self::ScannerFolder => "scanner.Folder",
|
||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
||||
Self::Dropped => "trace.Dropped",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TraceVal {
|
||||
Bool(bool),
|
||||
U64(u64),
|
||||
I64(i64),
|
||||
Str(Arc<str>),
|
||||
}
|
||||
|
||||
impl From<bool> for TraceVal {
|
||||
fn from(value: bool) -> Self {
|
||||
Self::Bool(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for TraceVal {
|
||||
fn from(value: u64) -> Self {
|
||||
Self::U64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for TraceVal {
|
||||
fn from(value: i64) -> Self {
|
||||
Self::I64(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for TraceVal {
|
||||
fn from(value: &str) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for TraceVal {
|
||||
fn from(value: String) -> Self {
|
||||
Self::Str(Arc::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceAttr {
|
||||
pub key: &'static str,
|
||||
pub value: TraceVal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TraceEvent {
|
||||
pub kind: TraceKind,
|
||||
pub func: TraceFunc,
|
||||
pub time: SystemTime,
|
||||
pub bucket: Option<Arc<str>>,
|
||||
pub object: Option<Arc<str>>,
|
||||
pub duration: Duration,
|
||||
pub bytes: u64,
|
||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
||||
}
|
||||
|
||||
impl TraceEvent {
|
||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
func,
|
||||
time: SystemTime::now(),
|
||||
bucket: None,
|
||||
object: None,
|
||||
duration: Duration::ZERO,
|
||||
bytes: 0,
|
||||
attrs: SmallVec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
||||
self.bucket = Some(bucket.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
||||
self.object = Some(object.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
||||
self.duration = duration;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
||||
self.bytes = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
||||
self.attrs.push(TraceAttr {
|
||||
key,
|
||||
value: value.into(),
|
||||
});
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceBus {
|
||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceBus {
|
||||
pub fn new(capacity: usize) -> Self {
|
||||
let capacity = capacity.max(1);
|
||||
let (sender, _receiver) = broadcast::channel(capacity);
|
||||
Self {
|
||||
sender,
|
||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn subscriber_count(&self) -> usize {
|
||||
self.subscriber_count.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> TraceSubscription {
|
||||
let receiver = self.sender.subscribe();
|
||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
||||
TraceSubscription {
|
||||
receiver,
|
||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
if self.subscriber_count() == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.sender.send(Arc::new(build())).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TraceBus {
|
||||
fn default() -> Self {
|
||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TraceSubscription {
|
||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
||||
subscriber_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl TraceSubscription {
|
||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
|
||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
||||
self.receiver.try_recv()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceSubscription {
|
||||
fn drop(&mut self) {
|
||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn global_trace_bus() -> &'static TraceBus {
|
||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
||||
}
|
||||
|
||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
||||
global_trace_bus().subscribe()
|
||||
}
|
||||
|
||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
||||
global_trace_bus().emit(build)
|
||||
}
|
||||
|
||||
pub fn trace_subscriber_count() -> usize {
|
||||
global_trace_bus().subscriber_count()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
#[test]
|
||||
fn trace_emit_skips_builder_without_subscribers() {
|
||||
let bus = TraceBus::new(4);
|
||||
let built = AtomicUsize::new(0);
|
||||
|
||||
let sent = bus.emit(|| {
|
||||
built.fetch_add(1, Ordering::Relaxed);
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
});
|
||||
|
||||
assert!(!sent);
|
||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trace_subscriber_receives_event() {
|
||||
let bus = TraceBus::new(4);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
assert!(bus.emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
||||
.with_bucket("bucket")
|
||||
.with_object("object")
|
||||
.with_duration(Duration::from_millis(7))
|
||||
.with_bytes(11)
|
||||
.with_attr("dry", true)
|
||||
}));
|
||||
|
||||
let event = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect("subscriber should receive emitted trace event");
|
||||
|
||||
assert_eq!(event.kind, TraceKind::Heal);
|
||||
assert_eq!(event.func, TraceFunc::HealObject);
|
||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
||||
assert_eq!(event.object.as_deref(), Some("object"));
|
||||
assert_eq!(event.duration, Duration::from_millis(7));
|
||||
assert_eq!(event.bytes, 11);
|
||||
assert_eq!(
|
||||
event.attrs.as_slice(),
|
||||
&[TraceAttr {
|
||||
key: "dry",
|
||||
value: TraceVal::Bool(true)
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_subscription_drop_decrements_count() {
|
||||
let bus = TraceBus::new(4);
|
||||
let subscription = bus.subscribe();
|
||||
|
||||
assert_eq!(bus.subscriber_count(), 1);
|
||||
drop(subscription);
|
||||
assert_eq!(bus.subscriber_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
||||
let bus = TraceBus::new(2);
|
||||
let mut subscription = bus.subscribe();
|
||||
|
||||
for index in 0_u64..4 {
|
||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
||||
}
|
||||
|
||||
let err = subscription
|
||||
.recv()
|
||||
.await
|
||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
||||
}
|
||||
}
|
||||
@@ -14,9 +14,8 @@
|
||||
|
||||
//! Shared backpressure policy type.
|
||||
//!
|
||||
//! The runtime backpressure implementation (byte-watermark pipes and
|
||||
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
||||
//! carries the watermark policy type that implementation shares.
|
||||
//! This module only carries the watermark policy; the admission primitive it
|
||||
//! projects into lives in `rustfs-io-core`.
|
||||
|
||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||
|
||||
|
||||
@@ -177,3 +177,31 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
||||
|
||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||
|
||||
/// Environment variable that toggles the MRF (mission repair feed) intent
|
||||
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
||||
/// unconsumed intents are replayed from the durable journal after a restart.
|
||||
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
||||
|
||||
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
||||
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
||||
|
||||
/// Environment variable for the MRF journal byte budget. The journal is
|
||||
/// compacted once its on-disk size crosses this bound.
|
||||
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
||||
|
||||
/// Environment variable for the MRF journal replay batch size (intents per
|
||||
/// replay push round).
|
||||
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
||||
|
||||
/// Default behavior keeps the MRF intent pipeline enabled.
|
||||
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
||||
|
||||
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
||||
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
||||
|
||||
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
||||
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Default MRF replay batch size.
|
||||
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
||||
|
||||
@@ -234,6 +234,31 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||
|
||||
/// Enable foreground PutObject request admission.
|
||||
///
|
||||
/// This is an experimental, default-off foreground write backpressure gate for
|
||||
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
||||
/// path and only the existing request counters are updated.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
||||
|
||||
/// Maximum foreground PutObject requests admitted concurrently per process.
|
||||
///
|
||||
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
||||
/// A value of `0` disables the gate even when the enable flag is present, so a
|
||||
/// partially configured rollout cannot reject every PUT.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
||||
|
||||
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
||||
///
|
||||
/// Once this timeout expires the request fails before body ingest/storage
|
||||
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
||||
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
||||
|
||||
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
||||
|
||||
/// Environment variable for minimum GetObject timeout in seconds.
|
||||
///
|
||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||
|
||||
@@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
||||
/// scanner's writer-side cache has the same map type).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or is
|
||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
||||
/// was cut (largest first).
|
||||
pub fn prefix_usage_in_cache(
|
||||
cache: &HashMap<String, DataUsageEntry>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
) -> Option<PrefixUsageQuery> {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
let root = if prefix.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
let child_prefix = format!("{root}/");
|
||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
||||
.children
|
||||
.iter()
|
||||
.filter_map(|child_key| {
|
||||
let child = cache.get(child_key)?;
|
||||
let child_flat = flatten_entry(cache, child, 1)?;
|
||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
||||
// slash marks a directory object and is display-only here.
|
||||
let name = child_key
|
||||
.strip_prefix(child_prefix.as_str())
|
||||
.unwrap_or(child_key.as_str())
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
Some(PrefixUsageEntry {
|
||||
prefix: name,
|
||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
sub_prefixes.sort_by(|left, right| {
|
||||
right
|
||||
.usage
|
||||
.size
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
let truncated = sub_prefixes.len() > max_entries;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
|
||||
Some(PrefixUsageQuery {
|
||||
usage,
|
||||
compacted: entry.compacted,
|
||||
truncated,
|
||||
sub_prefixes,
|
||||
})
|
||||
}
|
||||
|
||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
||||
|
||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
||||
return None;
|
||||
}
|
||||
let mut flattened = DataUsageEntry::default();
|
||||
if !flattened.checked_merge(root) {
|
||||
return None;
|
||||
}
|
||||
flattened.compacted = root.compacted;
|
||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
||||
// child edge pointing back at the root's own key is still terminated by
|
||||
// the visited set on first encounter.
|
||||
let mut visited: HashSet<&str> = HashSet::new();
|
||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
||||
while let Some((key, child_depth)) = pending.pop() {
|
||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let entry = cache.get(key)?;
|
||||
if !flattened.checked_merge(entry) {
|
||||
return None;
|
||||
}
|
||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
||||
}
|
||||
flattened.children.clear();
|
||||
Some(flattened)
|
||||
}
|
||||
|
||||
/// Flattened counters of one prefix subtree, as returned by
|
||||
/// [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageSummary {
|
||||
pub size: u64,
|
||||
pub objects: u64,
|
||||
pub versions: u64,
|
||||
pub delete_markers: u64,
|
||||
}
|
||||
|
||||
impl PrefixUsageSummary {
|
||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
||||
Self {
|
||||
size: entry.size as u64,
|
||||
objects: entry.objects as u64,
|
||||
versions: entry.versions as u64,
|
||||
delete_markers: entry.delete_markers as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add another set's counters into this one (entries are partitioned by
|
||||
/// set, so per-set results sum).
|
||||
pub fn merge(&mut self, other: &Self) {
|
||||
self.size = self.size.saturating_add(other.size);
|
||||
self.objects = self.objects.saturating_add(other.objects);
|
||||
self.versions = self.versions.saturating_add(other.versions);
|
||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||
}
|
||||
}
|
||||
|
||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct PrefixUsageEntry {
|
||||
pub prefix: String,
|
||||
pub usage: PrefixUsageSummary,
|
||||
}
|
||||
|
||||
/// Result of [`DataUsageCache::prefix_usage`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PrefixUsageQuery {
|
||||
pub usage: PrefixUsageSummary,
|
||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
||||
/// but no sub-prefix breakdown exists on disk.
|
||||
pub compacted: bool,
|
||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
||||
pub truncated: bool,
|
||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
||||
}
|
||||
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
@@ -997,6 +1148,21 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
||||
/// full counters instead of first-level sizes only).
|
||||
///
|
||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
||||
/// names come straight off the child keys — no reverse mapping exists or
|
||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
||||
/// which the `compacted` flag reports so callers can say why the
|
||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
||||
/// `max_entries` and was cut (largest first).
|
||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
||||
}
|
||||
|
||||
pub fn force_compact(&mut self, limit: usize) {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
@@ -1898,6 +2064,126 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
||||
/// distinct counters so aggregation is observable.
|
||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
||||
cache.replace(
|
||||
path,
|
||||
parent,
|
||||
DataUsageEntry {
|
||||
size,
|
||||
objects,
|
||||
versions,
|
||||
delete_markers,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
insert("bucket", "", 0, 0, 0, 0);
|
||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
||||
cache
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let root = cache
|
||||
.prefix_usage("bucket", "", 100)
|
||||
.expect("root query must find the bucket entry");
|
||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
||||
assert_eq!(root.usage.objects, 5);
|
||||
assert_eq!(root.usage.versions, 7);
|
||||
assert_eq!(root.usage.delete_markers, 2);
|
||||
assert!(!root.compacted);
|
||||
assert!(!root.truncated);
|
||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
||||
// each flattened to its own subtree total.
|
||||
let names: Vec<(&str, u64)> = root
|
||||
.sub_prefixes
|
||||
.iter()
|
||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
||||
.collect();
|
||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
|
||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
||||
assert_eq!(b.usage.size, 240);
|
||||
assert_eq!(b.usage.versions, 5);
|
||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["c", "d"]);
|
||||
|
||||
// Prefix slashes are normalized away.
|
||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
||||
assert_eq!(slashed.usage.size, 240);
|
||||
|
||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_reports_and_respects_truncation() {
|
||||
let cache = prefix_usage_fixture_cache();
|
||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_marks_compacted_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 999,
|
||||
objects: 9,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
||||
assert_eq!(compacted.usage.size, 999);
|
||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
||||
// whole query, not unbounded recursion.
|
||||
let mut cache = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b".to_string());
|
||||
}
|
||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
||||
// The unaffected sibling still answers.
|
||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
||||
|
||||
// A child key with no entry (dangling link) is rejected rather than
|
||||
// silently dropped: half a tree would under-report usage.
|
||||
let mut dangling = prefix_usage_fixture_cache();
|
||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
||||
entry.children.insert("bucket/b/ghost".to_string());
|
||||
}
|
||||
assert!(
|
||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
||||
"a dangling child link must be rejected"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
const TEST_PORT_RANGE: u16 = 40_000;
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
@@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
struct PortAllocatorGuard;
|
||||
|
||||
impl PortAllocatorGuard {
|
||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
loop {
|
||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
remove_stale_port_allocator_lock();
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PortAllocatorGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port() -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
||||
return;
|
||||
};
|
||||
let Ok(modified) = metadata.modified() else {
|
||||
return;
|
||||
};
|
||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn capture_command_logs(
|
||||
command: &mut Command,
|
||||
log_path: Option<&str>,
|
||||
@@ -508,10 +572,21 @@ impl RustFSTestEnvironment {
|
||||
/// Find an available port for the test
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let port = listener.local_addr()?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let mut next_port = read_next_test_port();
|
||||
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
drop(listener);
|
||||
return Ok(port);
|
||||
}
|
||||
}
|
||||
|
||||
Err("no available E2E test port found".into())
|
||||
}
|
||||
|
||||
/// Kill any existing RustFS processes
|
||||
|
||||
@@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end
|
||||
|
||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions.
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||
|
||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||
|
||||
@@ -30,10 +30,12 @@ use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput,
|
||||
DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput,
|
||||
GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput,
|
||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -88,6 +90,13 @@ const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
||||
"x-rustfs-source-replication-legalhold-timestamp",
|
||||
"x-minio-source-replication-legalhold-timestamp",
|
||||
];
|
||||
/// Wire prefix of the SSE-C passthrough replication transport headers
|
||||
/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a
|
||||
/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with
|
||||
/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO /
|
||||
/// generic S3, which silently discard unknown x-* headers.
|
||||
const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-";
|
||||
const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm";
|
||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||
|
||||
@@ -103,6 +112,9 @@ pub enum Operation {
|
||||
GetObject,
|
||||
HeadObject,
|
||||
DeleteObject,
|
||||
GetObjectTagging,
|
||||
PutObjectTagging,
|
||||
DeleteObjectTagging,
|
||||
ListObjectVersions,
|
||||
CreateMultipartUpload,
|
||||
UploadPart,
|
||||
@@ -149,6 +161,42 @@ impl ReplicationTimestampHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-proxy related headers observed on a request, journaled so proxy
|
||||
/// tests can assert the exact wire contract: the anti-loop marker present,
|
||||
/// the replication-check exemption absent, and the client SSE-C key family
|
||||
/// forwarded verbatim. The SSE-C key value itself is never retained — only
|
||||
/// its presence.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProxyHeaderSnapshot {
|
||||
pub source_proxy_request: Option<String>,
|
||||
pub replication_check: Option<String>,
|
||||
pub ssec_algorithm: Option<String>,
|
||||
pub ssec_key_present: bool,
|
||||
pub ssec_key_md5: Option<String>,
|
||||
/// Whether the request carried any `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header, so fail-closed tests can assert the
|
||||
/// sender really shipped the material a dropping target discarded.
|
||||
pub ssec_transport_present: bool,
|
||||
}
|
||||
|
||||
impl ProxyHeaderSnapshot {
|
||||
fn from_headers(headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"])
|
||||
.map(bounded_journal_value),
|
||||
replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"])
|
||||
.map(bounded_journal_value),
|
||||
ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"),
|
||||
ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value),
|
||||
ssec_transport_present: headers
|
||||
.keys()
|
||||
.any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Credential-free request metadata retained for deterministic assertions.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestRecord {
|
||||
@@ -163,6 +211,7 @@ pub struct RequestRecord {
|
||||
pub content_length: Option<u64>,
|
||||
pub consumed_bytes: Option<usize>,
|
||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
||||
pub proxy_headers: ProxyHeaderSnapshot,
|
||||
pub fault: Option<FaultAction>,
|
||||
}
|
||||
|
||||
@@ -178,6 +227,10 @@ struct ControlState {
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
assign_own_multipart_version_ids: bool,
|
||||
/// MinIO-like mode: silently discard non-whitelisted replication
|
||||
/// transport headers instead of storing them (see
|
||||
/// [`REPLICATION_SSE_TRANSPORT_PREFIX`]).
|
||||
drop_unlisted_replication_headers: bool,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
total_bytes: usize,
|
||||
@@ -199,6 +252,12 @@ struct ObjectVersion {
|
||||
delete_marker: bool,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
/// Object tags as ordered key/value pairs (PutObjectTagging replaces the
|
||||
/// whole set, DeleteObjectTagging clears it).
|
||||
tags: Vec<(String, String)>,
|
||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
||||
/// target behavior); empty when the drop mode discarded them.
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -208,6 +267,7 @@ struct MultipartState {
|
||||
version_id: String,
|
||||
content_type: Option<String>,
|
||||
metadata: Option<HashMap<String, String>>,
|
||||
replication_sse_headers: Vec<(String, String)>,
|
||||
parts: BTreeMap<i32, MultipartPart>,
|
||||
}
|
||||
|
||||
@@ -428,6 +488,15 @@ impl FakeS3Target {
|
||||
|
||||
/// Mint own version ids for the multipart path only — models a target
|
||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||
/// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C
|
||||
/// passthrough transport header instead of storing it. The default (off)
|
||||
/// models a RustFS target, which preserves the headers and echoes SSE-C
|
||||
/// evidence (`x-amz-server-side-encryption-customer-algorithm`) on
|
||||
/// HEAD/GET of the replica.
|
||||
pub fn drop_unlisted_replication_headers(&self, enabled: bool) {
|
||||
lock(&self.backend.store).drop_unlisted_replication_headers = enabled;
|
||||
}
|
||||
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
@@ -569,6 +638,7 @@ impl S3Access for FaultAccess {
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse().ok());
|
||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
||||
let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers());
|
||||
let fault = record_request(
|
||||
&self.control,
|
||||
operation,
|
||||
@@ -576,6 +646,7 @@ impl S3Access for FaultAccess {
|
||||
parsed,
|
||||
content_length,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
);
|
||||
if let Some(RequestFault {
|
||||
action: FaultAction::Status(status),
|
||||
@@ -615,6 +686,9 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
||||
"GetObject" => Operation::GetObject,
|
||||
"HeadObject" => Operation::HeadObject,
|
||||
"DeleteObject" => Operation::DeleteObject,
|
||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||
"UploadPart" => Operation::UploadPart,
|
||||
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
||||
@@ -630,6 +704,7 @@ fn record_request(
|
||||
parsed: ParsedRequest,
|
||||
content_length: Option<u64>,
|
||||
replication_timestamps: ReplicationTimestampHeaders,
|
||||
proxy_headers: ProxyHeaderSnapshot,
|
||||
) -> Option<RequestFault> {
|
||||
let mut state = lock(control);
|
||||
let action = parsed
|
||||
@@ -655,6 +730,7 @@ fn record_request(
|
||||
content_length,
|
||||
consumed_bytes: None,
|
||||
replication_timestamps,
|
||||
proxy_headers,
|
||||
fault: action.clone(),
|
||||
});
|
||||
action.map(|action| RequestFault { sequence, action })
|
||||
@@ -721,6 +797,15 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||
(&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::GetObjectTagging
|
||||
}
|
||||
(&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::PutObjectTagging
|
||||
}
|
||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
||||
Operation::DeleteObjectTagging
|
||||
}
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
@@ -788,6 +873,29 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
||||
Ok(version_id.to_string())
|
||||
}
|
||||
|
||||
/// Capture the SSE-C passthrough transport headers a replication PUT carried.
|
||||
/// Returns an empty set in the MinIO-like drop mode.
|
||||
fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> {
|
||||
if drop_unlisted {
|
||||
return Vec::new();
|
||||
}
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX))
|
||||
.filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// SSE-C evidence a RustFS-like target echoes for a stored passthrough
|
||||
/// replica: the customer algorithm restored from the transport headers.
|
||||
fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option<String> {
|
||||
version
|
||||
.replication_sse_headers
|
||||
.iter()
|
||||
.find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER)
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
|
||||
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
||||
header_value(headers, &SOURCE_ETAG_HEADERS)
|
||||
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
||||
@@ -1135,6 +1243,33 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option<
|
||||
Ok(version.clone())
|
||||
}
|
||||
|
||||
/// Replace (or clear, with an empty vec) the tag set of the addressed
|
||||
/// version, returning its version id. Mirrors `find_version` addressing:
|
||||
/// explicit version id or the latest version, delete markers rejected.
|
||||
fn set_version_tags(
|
||||
state: &mut StoreState,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
tags: Vec<(String, String)>,
|
||||
) -> S3Result<String> {
|
||||
// Resolve first (immutable) so the error paths match find_version.
|
||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
||||
let versions = state
|
||||
.buckets
|
||||
.get_mut(bucket)
|
||||
.expect("bucket existence checked by find_version")
|
||||
.objects
|
||||
.get_mut(key)
|
||||
.expect("key existence checked by find_version");
|
||||
let version = versions
|
||||
.iter_mut()
|
||||
.find(|version| version.version_id == resolved)
|
||||
.expect("version existence checked by find_version");
|
||||
version.tags = tags;
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl S3 for FakeBackend {
|
||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
@@ -1231,7 +1366,10 @@ impl S3 for FakeBackend {
|
||||
let input = req.input;
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||
let (assign_own, drop_unlisted) = {
|
||||
let state = lock(&self.store);
|
||||
(state.assign_own_version_ids, state.drop_unlisted_replication_headers)
|
||||
};
|
||||
let version_id = new_version_id(&headers, assign_own)?;
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
Some(value) => value,
|
||||
@@ -1248,6 +1386,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
};
|
||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1268,6 +1408,7 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectOutput {
|
||||
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
||||
@@ -1277,6 +1418,7 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
@@ -1291,6 +1433,7 @@ impl S3 for FakeBackend {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(HeadObjectOutput {
|
||||
content_length: Some(version.body.len() as i64),
|
||||
@@ -1299,12 +1442,79 @@ impl S3 for FakeBackend {
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
sse_customer_algorithm,
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version = {
|
||||
let state = lock(&self.store);
|
||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||
};
|
||||
let tag_set: TagSet = version
|
||||
.tags
|
||||
.into_iter()
|
||||
.map(|(key, value)| Tag {
|
||||
key: Some(key),
|
||||
value: Some(value),
|
||||
})
|
||||
.collect();
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(GetObjectTaggingOutput {
|
||||
tag_set,
|
||||
version_id: Some(ObjectVersionId::from(version.version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let tags = input
|
||||
.tagging
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default()))
|
||||
.collect();
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let input = req.input;
|
||||
let version_id = {
|
||||
let mut state = lock(&self.store);
|
||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())?
|
||||
};
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(DeleteObjectTaggingOutput {
|
||||
version_id: Some(ObjectVersionId::from(version_id)),
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
@@ -1381,6 +1591,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: true,
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
Ok(apply_response_fault(
|
||||
@@ -1408,9 +1620,10 @@ impl S3 for FakeBackend {
|
||||
ensure_upload_budget(&state)?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let upload_id = Uuid::new_v4().to_string();
|
||||
// Read the flag before the mutable borrow of `state.uploads` below
|
||||
// Read the flags before the mutable borrow of `state.uploads` below
|
||||
// (and never re-lock the store: the mutex is not reentrant).
|
||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||
let drop_unlisted = state.drop_unlisted_replication_headers;
|
||||
let version_id = new_version_id(&headers, mint_own)?;
|
||||
state.uploads.insert(
|
||||
upload_id.clone(),
|
||||
@@ -1420,6 +1633,7 @@ impl S3 for FakeBackend {
|
||||
version_id,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -1557,6 +1771,7 @@ impl S3 for FakeBackend {
|
||||
version_id: upload.version_id.clone(),
|
||||
content_type: upload.content_type.clone(),
|
||||
metadata: upload.metadata.clone(),
|
||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
selected,
|
||||
@@ -1583,6 +1798,8 @@ impl S3 for FakeBackend {
|
||||
delete_marker: false,
|
||||
content_type: upload.content_type,
|
||||
metadata: upload.metadata,
|
||||
tags: Vec::new(),
|
||||
replication_sse_headers: upload.replication_sse_headers,
|
||||
};
|
||||
let mut state = lock(&self.store);
|
||||
let current = state
|
||||
@@ -1787,6 +2004,65 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Default mode is RustFS-like: SSE-C passthrough transport headers are
|
||||
/// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is
|
||||
/// MinIO-like: the headers are silently discarded, so no evidence comes
|
||||
/// back — the exact difference the N2 fail-closed audit keys on. Both
|
||||
/// modes journal that the sender shipped the transport headers.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("target-bucket");
|
||||
let client = client(&target);
|
||||
|
||||
let put_with_transport_headers = |key: &'static str| {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("target-bucket")
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"ciphertext"))
|
||||
.customize()
|
||||
.map_request(move |mut request| {
|
||||
let headers = request.headers_mut();
|
||||
headers.insert("x-rustfs-replication-ssec-algorithm", "AES256");
|
||||
headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA==");
|
||||
Ok::<_, std::convert::Infallible>(request)
|
||||
})
|
||||
.send()
|
||||
};
|
||||
|
||||
put_with_transport_headers("kept").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), Some("AES256"));
|
||||
let get = client.get_object().bucket("target-bucket").key("kept").send().await?;
|
||||
assert_eq!(get.sse_customer_algorithm(), Some("AES256"));
|
||||
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
put_with_transport_headers("dropped").await?;
|
||||
let head = client.head_object().bucket("target-bucket").key("dropped").send().await?;
|
||||
assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence");
|
||||
|
||||
let requests = target.requests();
|
||||
for key in ["kept", "dropped"] {
|
||||
let record = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key))
|
||||
.expect("PUT must be journaled");
|
||||
assert!(
|
||||
record.proxy_headers.ssec_transport_present,
|
||||
"the journal must prove the sender shipped the transport headers for {key}"
|
||||
);
|
||||
}
|
||||
let plain_head = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == Operation::HeadObject)
|
||||
.expect("HEAD must be journaled");
|
||||
assert!(!plain_head.proxy_headers.ssec_transport_present);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! assert_sdk_error {
|
||||
($error:expr, $status:expr, $code:expr) => {{
|
||||
let error = &$error;
|
||||
@@ -3052,6 +3328,7 @@ mod tests {
|
||||
version_id: index.to_string(),
|
||||
content_type: None,
|
||||
metadata: None,
|
||||
replication_sse_headers: Vec::new(),
|
||||
parts: BTreeMap::new(),
|
||||
},
|
||||
);
|
||||
@@ -3074,6 +3351,7 @@ mod tests {
|
||||
},
|
||||
Some(0),
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
}
|
||||
let records = lock(&control).requests.clone();
|
||||
@@ -3096,6 +3374,7 @@ mod tests {
|
||||
},
|
||||
None,
|
||||
ReplicationTimestampHeaders::default(),
|
||||
ProxyHeaderSnapshot::default(),
|
||||
);
|
||||
{
|
||||
let bounded_records = lock(&bounded_control);
|
||||
|
||||
@@ -2610,17 +2610,20 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
assert_eq!(payload["Status"], "OK");
|
||||
assert_eq!(payload["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["ActiveMutation"], true);
|
||||
assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(payload["Targets"][0]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
// A RustFS target adopts the source version id, so the P1-19
|
||||
// version-identity probe passes.
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||
// A RustFS target preserves the SSE-C passthrough transport headers and
|
||||
// echoes the customer algorithm on the replication-check HEAD (N2).
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["SsecPassthrough"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
let target_client = target_env.create_s3_client();
|
||||
let versions = target_client
|
||||
@@ -4649,6 +4652,410 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// N2 (backlog#1675 P1-22): SSE-C passthrough replication to a target that
|
||||
/// silently drops the `X-Rustfs-Replication-*` transport headers (MinIO-like
|
||||
/// behavior, modeled by the fake target's drop mode) used to report COMPLETED
|
||||
/// while the replica had irrecoverably lost its decryption material — the red
|
||||
/// light this test was born failing on. Fail-closed contract now under test:
|
||||
/// the first attempt PUTs, HEAD-backs the replica, finds no SSE-C evidence,
|
||||
/// records the target Unsupported and reports FAILED; a second SSE-C object
|
||||
/// fails without any PUT reaching the target (capability cache, proven from
|
||||
/// the target journal); plaintext objects still replicate COMPLETED.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "ssec-drop-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "ssec-drop-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let put_ssec = |key: &'static str| {
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"ssec fail-closed payload"))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
};
|
||||
|
||||
// First SSE-C object: the audit must catch the dropped material.
|
||||
put_ssec("ssec-first.txt").await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "ssec-first.txt", "FAILED", true).await?;
|
||||
|
||||
let requests = target.take_requests();
|
||||
let first_put = requests
|
||||
.iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject && record.key.as_deref() == Some("ssec-first.txt"))
|
||||
.ok_or("the first SSE-C object must have been PUT (capability was Unknown)")?;
|
||||
assert!(
|
||||
first_put.proxy_headers.ssec_transport_present,
|
||||
"the replication PUT must have shipped the SSE-C transport headers the target then dropped"
|
||||
);
|
||||
assert!(
|
||||
requests.iter().any(|record| {
|
||||
record.operation == FakeTargetOperation::HeadObject
|
||||
&& record.key.as_deref() == Some("ssec-first.txt")
|
||||
&& record.sequence > first_put.sequence
|
||||
&& record.proxy_headers.replication_check.as_deref() == Some("true")
|
||||
}),
|
||||
"the post-PUT HEAD-back audit must have run through the replication-check channel; journal: {requests:?}"
|
||||
);
|
||||
|
||||
// Second SSE-C object: the cached Unsupported verdict fails it closed
|
||||
// before any PUT — including MRF retries of the first object.
|
||||
put_ssec("ssec-second.txt").await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "ssec-second.txt", "FAILED", true).await?;
|
||||
assert!(
|
||||
!target.requests().iter().any(|record| {
|
||||
record.operation == FakeTargetOperation::PutObject
|
||||
&& record.key.as_deref() != Some("plain-control.txt")
|
||||
&& record.proxy_headers.ssec_transport_present
|
||||
}),
|
||||
"no further SSE-C ciphertext may reach a target recorded Unsupported; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// The gate is scoped to SSE-C: plaintext replication keeps working.
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("plain-control.txt")
|
||||
.body(ByteStream::from_static(b"plaintext control payload"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, "plain-control.txt", "COMPLETED", false).await?;
|
||||
assert!(target.has_object(target_bucket, "plain-control.txt"));
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// N2 (backlog#1675 P1-22): the admin replication-check must expose the same
|
||||
/// verdict operators would otherwise only learn from failing SSE-C objects —
|
||||
/// an SsecPassthrough probe phase that fails with the machine-readable
|
||||
/// `BucketRemoteSsecPassthroughUnsupported` code against a header-dropping
|
||||
/// target, with no probe residue left behind. The target's overall status
|
||||
/// stays OK: unlike version-identity drift, dropped passthrough headers are
|
||||
/// a capability limit, and a plaintext-only deployment against a MinIO-like
|
||||
/// target must not turn red.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "ssec-check-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.drop_unlisted_replication_headers(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "ssec-check-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
|
||||
assert_eq!(
|
||||
payload["Status"], "OK",
|
||||
"a capability-only SSE-C failure must not fail the check overall: {payload}"
|
||||
);
|
||||
let target_report = &payload["Targets"][0];
|
||||
assert_eq!(target_report["Status"], "OK", "{payload}");
|
||||
let ssec = &target_report["Phases"]["SsecPassthrough"];
|
||||
assert_eq!(ssec["Status"], "FAILED", "SsecPassthrough phase must fail: {payload}");
|
||||
assert_eq!(
|
||||
ssec["Code"], "BucketRemoteSsecPassthroughUnsupported",
|
||||
"the failure must carry the machine-readable code: {payload}"
|
||||
);
|
||||
// Basic replication of plaintext objects works on this target: every other
|
||||
// phase passes, so the code is the discriminator operators branch on.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
// The SSE-C probe PUT must have shipped the real transport header names —
|
||||
// a mangled or missing header set would fail the phase for the wrong
|
||||
// reason and mask a working target.
|
||||
let requests = target.requests();
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::PutObject && record.proxy_headers.ssec_transport_present),
|
||||
"the SSE-C probe PUT must carry the X-Rustfs-Replication-* transport headers; journal: {requests:?}"
|
||||
);
|
||||
|
||||
// No probe residue, including the SSE-C probe version.
|
||||
let probe_put = requests
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.ok_or("the probe PUT never reached the fake target")?;
|
||||
let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?;
|
||||
assert!(
|
||||
target.stored_versions(target_bucket, &probe_key).is_empty(),
|
||||
"all probe versions must be cleaned up"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// C1 (backlog#1675 P1-22): heal-path convergence for SSE-C. An SSE-C object
|
||||
/// whose live replication failed during a target outage must converge through
|
||||
/// the scanner/heal compensation once the target returns — passing the N2
|
||||
/// HEAD-back audit against the recovered RustFS target — and the replica must
|
||||
/// be readable with the customer key.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, mut target_env, source_bucket, target_bucket) =
|
||||
build_sse_replication_pair("ssec-heal", false, false).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let key = "ssec-heal-contract.txt";
|
||||
let body = b"repl-22 ssec heal payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
|
||||
// Target outage: the SSE-C write cannot replicate.
|
||||
target_env.stop_server();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// The failure is observable on the source (SSE-C HEAD needs the key).
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let head = source_client
|
||||
.head_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
match head.replication_status().map(|status| status.as_str()) {
|
||||
Some("PENDING") | Some("FAILED") => break,
|
||||
other => {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("source SSE-C object never reported PENDING/FAILED; last status={other:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recover the target in place; the source scanner re-drives the failure.
|
||||
target_env
|
||||
.restart_server_preserving_data(vec![], &[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")])
|
||||
.await?;
|
||||
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", true).await?;
|
||||
|
||||
// The healed replica is a REPLICA (status surfaces on HEAD) readable with
|
||||
// the customer key.
|
||||
let target_client = target_env.create_s3_client();
|
||||
let replica_head = target_client
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica_head.replication_status().map(|status| status.as_str()),
|
||||
Some("REPLICA"),
|
||||
"the healed copy must carry REPLICA status"
|
||||
);
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// C1 (backlog#1675 P1-22): existing-object resync for SSE-C. An SSE-C object
|
||||
/// written BEFORE any replication config must reach the RustFS target through
|
||||
/// the existing-object resync (`replicate_all` transport, N2-audited), land as
|
||||
/// a REPLICA, and read back with the customer key.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||
source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
])
|
||||
.await?;
|
||||
|
||||
let source_bucket = "ssec-existing-src";
|
||||
let target_bucket = "ssec-existing-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
// The SSE-C object exists before any replication wiring.
|
||||
let key = "ssec-existing-contract.txt";
|
||||
let body = b"repl-22 ssec existing-object payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Wire replication (existing-object enabled) and drive a resync.
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
let (reset_arn, reset_id) = start_bucket_replication_reset(&source_env, source_bucket).await?;
|
||||
assert_eq!(reset_arn, target_arn);
|
||||
let terminal = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |status| {
|
||||
status.reset_id == reset_id && matches!(status.status.as_str(), "Completed" | "Failed")
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(terminal.status, "Completed", "SSE-C existing-object resync must complete");
|
||||
assert!(terminal.replicated_count >= 1, "the existing SSE-C object must have been resynced");
|
||||
|
||||
// The replica is a REPLICA (status surfaces on HEAD) readable with the
|
||||
// customer key.
|
||||
let replica_head = target_client
|
||||
.head_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
replica_head.replication_status().map(|status| status.as_str()),
|
||||
Some("REPLICA"),
|
||||
"the resynced copy must carry REPLICA status"
|
||||
);
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(replica.sse_customer_algorithm(), Some("AES256"));
|
||||
assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||
|
||||
// No plaintext leak: the replica stays unreadable without the key.
|
||||
assert!(
|
||||
target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.is_err(),
|
||||
"SSE-C replica must not be readable without the customer key"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#1147 repl-17 / backlog#1783: SSE-S3 objects replicate by decrypting
|
||||
/// at the source and re-encrypting on the target with the target's own KMS.
|
||||
/// The property backlog#1291 pinned — never a silent plaintext replica — still
|
||||
@@ -8417,3 +8824,304 @@ async fn test_scanner_never_compensates_when_existing_object_replication_disable
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shared setup for the P1-5 read-proxy scenarios (backlog#1675): a RustFS
|
||||
/// source with an enabled replication rule pointing at the fake target, and
|
||||
/// an object seeded DIRECTLY on the target — it exists remotely but not
|
||||
/// locally, exactly the active-active replication-lag window the read proxy
|
||||
/// serves.
|
||||
async fn start_read_proxy_lab(
|
||||
source_bucket: &str,
|
||||
target_bucket: &str,
|
||||
) -> Result<(FakeS3Target, RustFSTestEnvironment, Client, Client), Box<dyn Error + Send + Sync>> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(target_bucket);
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut process_env = replication_fast_env();
|
||||
process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
process_env.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
("RUST_LOG", "error"),
|
||||
]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &process_env).await?;
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let target_client = Client::from_conf(crate::common::build_test_s3_config(
|
||||
target.endpoint(),
|
||||
FAKE_ACCESS_KEY,
|
||||
FAKE_SECRET_KEY,
|
||||
None,
|
||||
"read-proxy-e2e",
|
||||
));
|
||||
|
||||
Ok((target, source_env, source_client, target_client))
|
||||
}
|
||||
|
||||
/// P1-5 (backlog#1675): during the active-active replication lag window a
|
||||
/// GET/HEAD for an object the local site does not have yet is proxied to the
|
||||
/// replication target. Pins the wire contract: the anti-loop
|
||||
/// `source-proxy-request` marker is sent, the replication worker's
|
||||
/// `source-replication-check` SSE-C exemption is NEVER sent, client SSE-C
|
||||
/// headers are forwarded verbatim, and an inbound request that was itself
|
||||
/// proxied is answered locally (404) without touching the target.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let source_bucket = "proxy-read-src";
|
||||
let target_bucket = "proxy-read-dst";
|
||||
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||
|
||||
let payload = b"proxy payload".to_vec();
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-only")
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
target.take_requests();
|
||||
|
||||
// a. GET of the locally-missing object is served through the proxy.
|
||||
let got = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied GET failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(got.content_length, Some(payload.len() as i64));
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), payload.as_slice(), "proxied GET must stream the target's body");
|
||||
|
||||
let get_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied GET")?;
|
||||
assert_eq!(
|
||||
get_record.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("true"),
|
||||
"proxied GET must carry the anti-loop source-proxy-request marker"
|
||||
);
|
||||
assert!(
|
||||
get_record.proxy_headers.replication_check.is_none(),
|
||||
"proxied GET must never carry the replication worker's source-replication-check exemption"
|
||||
);
|
||||
assert!(
|
||||
get_record.proxy_headers.ssec_algorithm.is_none() && !get_record.proxy_headers.ssec_key_present,
|
||||
"no client SSE-C headers were sent, so none may be forwarded"
|
||||
);
|
||||
|
||||
// a2. Client SSE-C headers travel verbatim to the target (the target owns
|
||||
// the real SSE-C decryption; the plaintext fake simply ignores them).
|
||||
target.take_requests();
|
||||
let ssec_key = "01234567890123456789012345678901";
|
||||
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
|
||||
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
|
||||
let _ = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&ssec_key_b64)
|
||||
.sse_customer_key_md5(&ssec_key_md5)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied SSE-C GET failed: {}", err.into_service_error()))?;
|
||||
let ssec_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied SSE-C GET")?;
|
||||
assert_eq!(ssec_record.proxy_headers.ssec_algorithm.as_deref(), Some("AES256"));
|
||||
assert!(ssec_record.proxy_headers.ssec_key_present, "SSE-C key header must be forwarded verbatim");
|
||||
assert_eq!(ssec_record.proxy_headers.ssec_key_md5.as_deref(), Some(ssec_key_md5.as_str()));
|
||||
assert!(ssec_record.proxy_headers.replication_check.is_none());
|
||||
|
||||
// b. HEAD of the locally-missing object is served through the proxy.
|
||||
target.take_requests();
|
||||
let head = source_client
|
||||
.head_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied HEAD failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(head.content_length, Some(payload.len() as i64));
|
||||
let head_record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("proxy-only"))
|
||||
.ok_or("fake target never received the proxied HEAD")?;
|
||||
assert_eq!(head_record.proxy_headers.source_proxy_request.as_deref(), Some("true"));
|
||||
assert!(head_record.proxy_headers.replication_check.is_none());
|
||||
|
||||
// c. Anti-loop: an inbound request that already carries the proxy marker
|
||||
// is answered locally with 404 and never forwarded to the target.
|
||||
target.take_requests();
|
||||
let err = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-minio-source-proxy-request", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("anti-loop GET must fail locally instead of proxying");
|
||||
let service_err = err.into_service_error();
|
||||
assert!(service_err.is_no_such_key(), "anti-loop GET must 404, got: {service_err}");
|
||||
assert!(
|
||||
!target
|
||||
.requests()
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||
"anti-loop GET must not reach the replication target; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// c2. MinIO ProxyHeaderSet parity: the header's mere PRESENCE disables
|
||||
// proxying — "false" is exactly what a peer's replication worker sends on
|
||||
// its convergence HEADs, and proxying that miss back would fake
|
||||
// convergence.
|
||||
target.take_requests();
|
||||
let err = source_client
|
||||
.get_object()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-only")
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-minio-source-proxy-request", "false");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("proxy-header-set GET must fail locally instead of proxying");
|
||||
let service_err = err.into_service_error();
|
||||
assert!(service_err.is_no_such_key(), "proxy-header-set GET must 404, got: {service_err}");
|
||||
assert!(
|
||||
!target
|
||||
.requests()
|
||||
.iter()
|
||||
.any(|record| record.operation == FakeTargetOperation::GetObject),
|
||||
"proxy-header-set GET must not reach the replication target; journal: {:?}",
|
||||
target.requests()
|
||||
);
|
||||
|
||||
// d. The replication worker's own convergence HEAD against the target
|
||||
// must carry `source-proxy-request: false` (never proxied back) and the
|
||||
// replication-check exemption. Trigger real replication and inspect the
|
||||
// fake journal.
|
||||
target.take_requests();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("worker-replicated")
|
||||
.body(ByteStream::from_static(b"worker payload"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "worker-replicated").await?;
|
||||
let worker_head = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::HeadObject && record.key.as_deref() == Some("worker-replicated"))
|
||||
.ok_or_else(|| format!("replication worker never HEAD-ed the target; journal: {:?}", target.requests()))?;
|
||||
assert_eq!(
|
||||
worker_head.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("false"),
|
||||
"worker convergence HEAD must send source-proxy-request: false so the target answers locally"
|
||||
);
|
||||
assert_eq!(
|
||||
worker_head.proxy_headers.replication_check.as_deref(),
|
||||
Some("true"),
|
||||
"worker convergence HEAD keeps the replication-check exemption"
|
||||
);
|
||||
|
||||
drop(source_env);
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-5 (backlog#1675): GetObjectTagging for an object missing locally is
|
||||
/// proxied to the replication target with the anti-loop marker, mirroring
|
||||
/// MinIO `proxyGetTaggingToRepTarget`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let source_bucket = "proxy-tag-src";
|
||||
let target_bucket = "proxy-tag-dst";
|
||||
let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
|
||||
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-tagged")
|
||||
.body(ByteStream::from_static(b"tagged payload"))
|
||||
.send()
|
||||
.await?;
|
||||
target_client
|
||||
.put_object_tagging()
|
||||
.bucket(target_bucket)
|
||||
.key("proxy-tagged")
|
||||
.tagging(
|
||||
aws_sdk_s3::types::Tagging::builder()
|
||||
.tag_set(aws_sdk_s3::types::Tag::builder().key("team").value("storage").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
target.take_requests();
|
||||
|
||||
let tags = source_client
|
||||
.get_object_tagging()
|
||||
.bucket(source_bucket)
|
||||
.key("proxy-tagged")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("proxied GetObjectTagging failed: {}", err.into_service_error()))?;
|
||||
assert_eq!(tags.tag_set.len(), 1, "proxied tagging read must return the target's tags");
|
||||
assert_eq!(tags.tag_set[0].key.as_str(), "team");
|
||||
assert_eq!(tags.tag_set[0].value.as_str(), "storage");
|
||||
|
||||
let record = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::GetObjectTagging && record.key.as_deref() == Some("proxy-tagged"))
|
||||
.ok_or("fake target never received the proxied GetObjectTagging")?;
|
||||
assert_eq!(
|
||||
record.proxy_headers.source_proxy_request.as_deref(),
|
||||
Some("true"),
|
||||
"proxied tagging read must carry the anti-loop marker"
|
||||
);
|
||||
assert!(record.proxy_headers.replication_check.is_none());
|
||||
|
||||
drop(source_env);
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
TargetClient, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -198,12 +198,13 @@ pub mod bucket {
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -373,14 +374,14 @@ pub mod error {
|
||||
|
||||
pub mod erasure {
|
||||
pub use crate::erasure::coding::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
|
||||
calc_shard_size, calc_shard_size_legacy,
|
||||
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError,
|
||||
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod event {
|
||||
pub use crate::event::name::EventName;
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||
pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event};
|
||||
}
|
||||
|
||||
pub mod global {
|
||||
@@ -483,6 +484,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
|
||||
@@ -27,10 +27,15 @@ use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
use aws_sdk_s3::operation::delete_object_tagging::{DeleteObjectTaggingError, DeleteObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
|
||||
use aws_sdk_s3::operation::get_object_tagging::{GetObjectTaggingError, GetObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::head_bucket::HeadBucketError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
};
|
||||
@@ -57,8 +62,8 @@ use rustfs_utils::http::{
|
||||
is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
|
||||
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
};
|
||||
@@ -294,9 +299,41 @@ struct TargetClientBuildProbe {
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
/// SSE-C passthrough capability verdicts (see the enum's own docs in
|
||||
/// `rustfs-replication`) are cached here per target ARN: entries follow the
|
||||
/// `arn_remotes_map` lifecycle (rebuilding or removing a target resets its
|
||||
/// capability to `Unknown`) and additionally expire after
|
||||
/// [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], after which the next attempt
|
||||
/// re-audits. Re-exported so existing `bucket_target_sys` consumers keep
|
||||
/// their import path while the verdict vocabulary lives with the
|
||||
/// replication decision logic.
|
||||
pub use crate::bucket::replication::SsecPassthroughCapability;
|
||||
|
||||
/// How long an audited SSE-C passthrough verdict stays authoritative.
|
||||
///
|
||||
/// Trade-off: without a TTL a verdict is sticky for the process lifetime —
|
||||
/// an `Unsupported` target that gets upgraded (or re-probed only via
|
||||
/// replication-check) would keep failing SSE-C replication forever, and the
|
||||
/// fail-open twin: a `Supported` verdict would outlive a backend swapped
|
||||
/// behind the same endpoint/ARN. With the TTL, a bad target costs at most
|
||||
/// one wasted PUT+HEAD audit per TTL window, and a changed backend is
|
||||
/// re-discovered within the same window.
|
||||
pub const SSEC_PASSTHROUGH_CAPABILITY_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
/// A recorded SSE-C passthrough verdict plus when it was recorded, so reads
|
||||
/// can report staleness against [`SSEC_PASSTHROUGH_CAPABILITY_TTL`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct SsecPassthroughRecord {
|
||||
capability: SsecPassthroughCapability,
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
/// SSE-C passthrough capability verdicts keyed by target ARN. See
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
@@ -317,6 +354,7 @@ impl BucketTargetSys {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -580,19 +618,59 @@ impl BucketTargetSys {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
|
||||
if let Some(targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
ssec_map.remove(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached SSE-C passthrough capability for a target ARN, plus whether the
|
||||
/// verdict is older than [`SSEC_PASSTHROUGH_CAPABILITY_TTL`]. `(Unknown,
|
||||
/// false)` when no verdict has been recorded since the target was built.
|
||||
/// Staleness is computed here so the gate policy stays a pure function.
|
||||
pub async fn ssec_passthrough_capability(&self, arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
match self.ssec_passthrough_map.read().await.get(arn) {
|
||||
Some(record) => (record.capability, record.recorded_at.elapsed() >= SSEC_PASSTHROUGH_CAPABILITY_TTL),
|
||||
None => (SsecPassthroughCapability::Unknown, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record an audited SSE-C passthrough verdict for a target ARN. Written by
|
||||
/// the replication worker's HEAD-back audit and by the replication-check
|
||||
/// SsecPassthrough probe phase.
|
||||
pub async fn record_ssec_passthrough_capability(&self, arn: &str, capability: SsecPassthroughCapability) {
|
||||
self.ssec_passthrough_map.write().await.insert(
|
||||
arn.to_string(),
|
||||
SsecPassthroughRecord {
|
||||
capability,
|
||||
recorded_at: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Test hook: age an existing verdict so TTL expiry is observable without
|
||||
/// waiting out the real window.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn backdate_ssec_passthrough_capability(&self, arn: &str, age: Duration) {
|
||||
let backdated = Instant::now()
|
||||
.checked_sub(age)
|
||||
.expect("system uptime must exceed the backdate age");
|
||||
if let Some(record) = self.ssec_passthrough_map.write().await.get_mut(arn) {
|
||||
record.recorded_at = backdated;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set_target(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -948,15 +1026,21 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
let mut ssec_map = self.ssec_passthrough_map.write().await;
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
// A rebuilt/edited target may point at a different service:
|
||||
// the SSE-C passthrough verdict must be re-audited from Unknown.
|
||||
ssec_map.remove(&target.arn);
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -1446,6 +1530,43 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the S3 `versionId` for a proxied read against a remote target.
|
||||
/// RustFS represents the null version internally as the nil UUID while the S3
|
||||
/// API addresses it as the literal "null" (same mapping as
|
||||
/// [`resolve_put_api_version_id`]); empty means "no version requested".
|
||||
pub(crate) fn resolve_read_api_version_id(version_id: Option<String>) -> Option<String> {
|
||||
let version_id = version_id?;
|
||||
let trimmed = version_id.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else if Uuid::parse_str(trimmed).is_ok_and(|uuid| uuid.is_nil()) {
|
||||
Some(rustfs_filemeta::NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Outbound header set for a proxied read: the caller-provided passthrough
|
||||
/// headers (client SSE-C key family, conditional headers) plus the anti-loop
|
||||
/// `source-proxy-request` marker in both the x-rustfs- and x-minio- prefixes
|
||||
/// (a MinIO target only understands the latter). Never adds
|
||||
/// `source-replication-check`: that exemption channel belongs exclusively to
|
||||
/// the replication worker's HEAD.
|
||||
fn proxy_outbound_headers(mut extra_headers: HeaderMap) -> HeaderMap {
|
||||
insert_header(&mut extra_headers, SUFFIX_SOURCE_PROXY_REQUEST, "true");
|
||||
extra_headers
|
||||
}
|
||||
|
||||
/// Copy `headers` onto an SDK request inside `customize().map_request` (runs
|
||||
/// before signing, so the headers join the SigV4 canonical request).
|
||||
fn apply_extra_headers(mut req: HttpRequest, headers: &HeaderMap) -> Result<HttpRequest, std::convert::Infallible> {
|
||||
for (k, v) in headers.iter() {
|
||||
req.headers_mut()
|
||||
.insert(k.as_str().to_string(), v.to_str().unwrap_or("").to_string());
|
||||
}
|
||||
Ok(req)
|
||||
}
|
||||
|
||||
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
||||
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
@@ -1549,8 +1670,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1561,6 +1682,7 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
@@ -1696,6 +1818,7 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, _c: Arc<TargetClient>) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
@@ -1851,6 +1974,13 @@ impl TargetClient {
|
||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
// `source-proxy-request: false` (MinIO `ProxyHeaderSet` semantics):
|
||||
// the header's mere presence tells the receiver to answer LOCALLY
|
||||
// instead of proxying the miss back to us. Without it, a not-found on
|
||||
// the target gets read-proxied back to this source, echoes the source
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
match self
|
||||
.client
|
||||
.head_object()
|
||||
@@ -1875,6 +2005,129 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
/// replicated locally, MinIO `proxyHeadToRepTarget`).
|
||||
///
|
||||
/// Deliberately different from [`TargetClient::head_object`]: it must NOT
|
||||
/// send `source-replication-check` — that header is the replication
|
||||
/// worker's SSE-C metadata exemption channel. A proxied client request
|
||||
/// instead forwards the client's own SSE-C headers (`extra_headers`) so
|
||||
/// the target performs the real SSE-C validation/decryption. The
|
||||
/// `source-proxy-request` marker is always added so the target does not
|
||||
/// proxy the request onward (anti-loop).
|
||||
pub async fn head_object_for_proxy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||
/// Returns the streaming SDK output; callers must forward the body without
|
||||
/// buffering it. Same header contract as [`Self::head_object_for_proxy`]:
|
||||
/// anti-loop marker on, replication-check never sent, client SSE-C /
|
||||
/// conditional headers forwarded verbatim via `extra_headers`.
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.set_range(range)
|
||||
.set_part_number(part_number)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// GetObjectTagging for the tagging read-proxy path
|
||||
/// (MinIO `proxyGetTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn get_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// PutObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn put_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
tagging: SdkTagging,
|
||||
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.tagging(tagging)
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging for the tagging proxy path
|
||||
/// (MinIO `proxyTaggingToRepTarget`). Anti-loop marker always added.
|
||||
pub async fn delete_object_tagging(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.delete_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
@@ -2504,6 +2757,57 @@ mod tests {
|
||||
assert_eq!(health.last_online, Some(now));
|
||||
}
|
||||
|
||||
/// N2 TTL contract, both flip directions: a recorded verdict is fresh
|
||||
/// until [`SSEC_PASSTHROUGH_CAPABILITY_TTL`], then reads as expired; a
|
||||
/// re-audit that records the OPPOSITE verdict replaces it as fresh. The
|
||||
/// worker gate maps expired verdicts to ProceedWithAudit (pinned in
|
||||
/// `replication_target_boundary`), so together this proves an Unsupported
|
||||
/// target recovers to Supported through the audit once its verdict ages
|
||||
/// out — and a stale Supported one is re-proven rather than trusted.
|
||||
#[tokio::test]
|
||||
async fn ssec_passthrough_capability_ttl_expires_and_reaudit_flips_verdict() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:ssec-ttl";
|
||||
let expired_age = SSEC_PASSTHROUGH_CAPABILITY_TTL + Duration::from_secs(1);
|
||||
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unknown, false),
|
||||
"an unrecorded target must read Unknown and never expired"
|
||||
);
|
||||
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Unsupported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, false)
|
||||
);
|
||||
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Unsupported, true),
|
||||
"an aged-out Unsupported verdict must read expired so the gate re-audits"
|
||||
);
|
||||
|
||||
// The re-audit against an upgraded target records Supported afresh.
|
||||
sys.record_ssec_passthrough_capability(arn, SsecPassthroughCapability::Supported)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, false),
|
||||
"a fresh Supported verdict replaces the expired Unsupported one"
|
||||
);
|
||||
|
||||
// And the fail-open twin: Supported also ages out.
|
||||
sys.backdate_ssec_passthrough_capability(arn, expired_age).await;
|
||||
assert_eq!(
|
||||
sys.ssec_passthrough_capability(arn).await,
|
||||
(SsecPassthroughCapability::Supported, true),
|
||||
"an aged-out Supported verdict must read expired so the gate re-proves it"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
@@ -456,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ExpiryStats {
|
||||
pub fn missed_tasks(&self) -> i64 {
|
||||
self.missed_expiry_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_free_vers_tasks(&self) -> i64 {
|
||||
self.missed_freevers_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn missed_tier_journal_tasks(&self) -> i64 {
|
||||
self.missed_tier_journal_tasks.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -1776,7 +1783,7 @@ impl TransitionState {
|
||||
.await;
|
||||
}
|
||||
global_metrics().record_scanner_transition_failed(1);
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
|
||||
if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
|
||||
error!(
|
||||
event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
|
||||
@@ -80,7 +80,10 @@ impl LastDayTierStats {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
|
||||
let mut cl = self.clone();
|
||||
let mut cm = m;
|
||||
|
||||
@@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
struct ObjSweeper {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object: String,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
bucket: String,
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
@@ -191,9 +192,9 @@ struct ObjSweeper {
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ObjSweeper {
|
||||
#[allow(clippy::new_ret_no_self)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
|
||||
Ok(Self {
|
||||
object: object.into(),
|
||||
@@ -202,17 +203,20 @@ impl ObjSweeper {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
|
||||
self.versioned = versioned;
|
||||
self.suspended = suspended;
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id.clone(),
|
||||
@@ -226,6 +230,7 @@ impl ObjSweeper {
|
||||
opts
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn set_transition_state(&mut self, info: TransitionedObject) {
|
||||
self.transition_tier = info.tier;
|
||||
self.transition_status = info.status;
|
||||
@@ -266,6 +271,7 @@ impl ObjSweeper {
|
||||
None
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub async fn sweep(&self, api: Arc<ECStore>) {
|
||||
let Some(je) = self.should_remove_remote_object() else {
|
||||
return;
|
||||
|
||||
@@ -312,9 +312,7 @@ mod tests {
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyBucketQuota {
|
||||
#[allow(dead_code)]
|
||||
quota: Option<u64>,
|
||||
#[allow(dead_code)]
|
||||
quota_type: LegacyQuotaType,
|
||||
}
|
||||
let legacy = serde_json::from_slice::<LegacyBucketQuota>(&json)
|
||||
|
||||
@@ -11,9 +11,9 @@ paths.
|
||||
| Module | Current role | Split blocker |
|
||||
|---|---|---|
|
||||
| `config.rs` | Replication config helpers, rule matching, and tag filtering. | Uses replication-local filemeta/tagging boundaries and S3 DTOs directly. |
|
||||
| `datatypes.rs` | ECStore compatibility re-export for resync status enums. | Re-exports `rustfs-replication` contracts while downstream facade consumers migrate. |
|
||||
| `replication_object_decision_boundary.rs` | Object replication option DTOs, resync target projection, delete replication decisions, and multipart planning helpers. | Keeps ECStore runtime modules from importing object decision contracts directly from `rustfs-replication`. |
|
||||
| `replication_pool.rs` | Replication queue, worker pool, MRF persistence, bucket stats, and delete/object scheduling. | Depends on bucket target sys, bucket metadata sys, metadata paths, queue contracts through the queue boundary, file metadata replication contracts through local boundaries, config storage, storage contracts through the replication storage boundary, runtime sources, and notification state. |
|
||||
| `replication_proxy.rs` | Proxy-target selection for GET/HEAD/Tagging reads of objects not yet replicated locally (MinIO `getProxyTargets` parity: anti-loop, version-suspended, and no-config empty branches). | Uses replication config lookup, rule matching, and target clients through local boundaries. |
|
||||
| `replication_queue_boundary.rs` | Queue/admission DTOs, heal queue DTOs, worker sizing, and backpressure helpers. | Keeps ECStore runtime modules from importing queue/backpressure contracts directly from `rustfs-replication`. |
|
||||
| `replication_resync_boundary.rs` | Resync DTOs, status classifiers, persisted resync/MRF codec wrappers, and ECStore error mapping. | Keeps ECStore runtime modules from importing resync contract helpers directly from `rustfs-replication`. |
|
||||
| `replication_resyncer.rs` | Object replication, delete replication, resync execution, target calls, and multipart target upload paths. | Depends on target calls and target config types through the replication target boundary, metadata paths and metadata systems through the replication metadata boundary, file metadata replication contracts through the filemeta boundary, object decisions and multipart planning through the object decision boundary, resync contracts through the resync boundary, queue DTOs through the queue boundary, error contracts through the error boundary, versioning systems, storage contracts through the replication storage boundary, config-derived storage class labels through the config store, runtime sources, notification events and local event host selection through the event sink, bandwidth reader wrapping, and SetDisks lock timing. |
|
||||
@@ -117,9 +117,12 @@ Target end state:
|
||||
their file names — so batch-merging them beforehand is explicitly rejected:
|
||||
it forces synchronized guard-script/mod/import churn with zero functional
|
||||
gain;
|
||||
- the only module that can retire early is `datatypes.rs`: delete it once its
|
||||
facade consumers import the resync status enums through `rustfs-replication`
|
||||
directly.
|
||||
- `datatypes.rs` retired early (its sanctioned exception): it was a pure
|
||||
relay (`boundary -> datatypes -> mod.rs`), so the facade now re-exports
|
||||
`ResyncStatusType` from the resync boundary directly and the relay file is
|
||||
deleted. Note the original retirement wording ("consumers import through
|
||||
`rustfs-replication` directly") conflicted with Migration Rule #15 —
|
||||
consumers stay behind the ECStore facade; only the relay hop dissolves.
|
||||
|
||||
## Milestones
|
||||
|
||||
@@ -127,9 +130,9 @@ Target end state:
|
||||
|---|---|---|
|
||||
| M0 | Record the completion criteria and end state (this section). | Done |
|
||||
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Done — moved the pure decision helpers with their unit tests: `resync_status_duration` (resync), `resync_existing_delete_replication_info` / `replicate_delete_outcome` / `target_delete_version_id` / `delete_marker_purge_version_id` / `delete_marker_purge_mrf_entry` (delete), `version_identity_drifted` / `is_replication_target_offline_error` / the SSE-C passthrough gate family incl. `SsecPassthroughCapability` (object; `ssec_passthrough_evidence_present` was param-demoted to the echoed customer-algorithm string, ECStore keeps the `HeadObjectOutput` adapter). ECStore imports them through the resync/object-decision/target boundaries; `bucket_target_sys` keeps only the verdict cache + TTL and re-exports the capability enum. Not moved (signatures carry ECStore or aws-sdk types): `verify_resync_head_result`, `resync_target_error_detail`, the `SdkError` classifiers (`has_raw_status`, `is_version_id_format_mismatch`), the `replicate_all_*` option/info builders, and `bounded_resync_max_jobs` (itself a pure clamp, but it forms one local configuration unit with the env-reading `configured_resync_max_jobs` and its ECStore-local constants — moving the clamp alone has negative value). |
|
||||
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries. | Pending (`datatypes.rs` already retired early alongside M2) |
|
||||
|
||||
The original first code-bearing step (narrow `ReplicationEventSink` /
|
||||
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod datatypes;
|
||||
mod replication_bandwidth_boundary;
|
||||
mod replication_config_boundary;
|
||||
mod replication_config_store;
|
||||
@@ -29,6 +28,7 @@ mod replication_object_bridge;
|
||||
mod replication_object_config;
|
||||
mod replication_object_decision_boundary;
|
||||
pub(crate) mod replication_pool;
|
||||
mod replication_proxy;
|
||||
mod replication_queue_boundary;
|
||||
mod replication_resync_boundary;
|
||||
mod replication_resyncer;
|
||||
@@ -43,7 +43,6 @@ pub(crate) mod replication_timing;
|
||||
mod replication_versioning_boundary;
|
||||
mod runtime_boundary;
|
||||
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
@@ -74,13 +73,16 @@ pub use replication_pool::{
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_proxy::get_proxy_targets;
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
};
|
||||
pub use replication_resync_boundary::ResyncStatusType;
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -12,12 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_filemeta::NULL_VERSION_ID;
|
||||
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
||||
parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
REPLICATE_EXISTING, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
ReplicationWorkerOperation, ResyncDecision, get_replication_state, parse_replicate_decision,
|
||||
replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
pub use rustfs_replication::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
|
||||
@@ -18,9 +18,10 @@ pub use rustfs_replication::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject,
|
||||
delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object,
|
||||
should_retry_delete_marker_purge,
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -667,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => Some(guard),
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and decodes the on-disk MRF recovery file.
|
||||
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return None;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match decode_mrf_file(&data) {
|
||||
Ok(v) => Some(v),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(storage, &data).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays one MRF recovery entry by operation kind.
|
||||
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||
}
|
||||
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||
/// stale force-delete generations are skipped, and plain deletes are
|
||||
/// reconstructed as heal deletes.
|
||||
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let operation_id = entry.force_delete_id?;
|
||||
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
Some(ReplicationQueueAdmission::Skipped)
|
||||
} else {
|
||||
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
/// get_object_info here because the delete-marker or version may
|
||||
/// already be absent from the local store — that is expected.
|
||||
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The MRF entry does not persist the replication decision and the
|
||||
/// source object is gone, so re-derive the decision from the live
|
||||
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
/// it on the reconstructed delete. Without this the decision string
|
||||
/// is empty and the delete replicates to zero targets — a silent
|
||||
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
async fn resolve_mrf_delete_replicate_decision(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
versioned: bool,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicateDecision> {
|
||||
if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => None,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => Some(dsc),
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
None
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||
/// replication decision.
|
||||
fn reconstructed_heal_delete_info(
|
||||
entry: &MrfReplicateEntry,
|
||||
oi: &ObjectInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replays a metadata-kind MRF entry against the live source object.
|
||||
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||
entry: &MrfReplicateEntry,
|
||||
storage: &Arc<S>,
|
||||
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||
) -> Option<ReplicationQueueAdmission> {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||
} else {
|
||||
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
Some(ReplicationQueueAdmission::Queued)
|
||||
} else {
|
||||
Some(ReplicationQueueAdmission::Missed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||
/// admitted targets, carrying over the entry's retry count.
|
||||
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
roi
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
entries: &[MrfReplicateEntry],
|
||||
retry_entries: &[MrfReplicateEntry],
|
||||
) -> Vec<MrfReplicateEntry> {
|
||||
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.to_vec()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1221,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let recovery_lock = match storage
|
||||
.new_ns_lock(
|
||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(lock) => lock,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to create the MRF recovery leader lock"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let recovery_guard = match recovery_lock
|
||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
"Another node is already processing the MRF recovery backlog"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to load MRF recovery file"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let entries = match decode_mrf_file(&data) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %e,
|
||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||
);
|
||||
quarantine_mrf_file(&storage, &data).await;
|
||||
return;
|
||||
}
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
@@ -1294,187 +1597,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let admission = match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
if should_replay_force_delete_intent(entry) {
|
||||
let Some(operation_id) = entry.force_delete_id else {
|
||||
continue;
|
||||
};
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
force_delete: true,
|
||||
force_delete_id: Some(operation_id),
|
||||
force_delete_target_arns: entry.target_arns.clone(),
|
||||
force_delete_generation: entry.force_delete_generation,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
} else if entry.force_delete_id.is_some() {
|
||||
ReplicationQueueAdmission::Skipped
|
||||
} else {
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
//
|
||||
// The MRF entry does not persist the replication decision and the
|
||||
// source object is gone, so re-derive the decision from the live
|
||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||
// it on the reconstructed delete. Without this the decision string
|
||||
// is empty and the delete replicates to zero targets — a silent
|
||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = if entry.target_arns.is_empty() {
|
||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||
Ok(None) => continue,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dsc) => dsc,
|
||||
Err(_) => {
|
||||
retry_entries.push(entry.clone());
|
||||
continue;
|
||||
}
|
||||
},
|
||||
}
|
||||
} else {
|
||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
||||
};
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
|
||||
let delete_marker_mtime = entry
|
||||
.delete_marker_mtime
|
||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
force_delete: entry.force_delete,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: entry.bucket.clone(),
|
||||
op_type: ReplicationType::Heal,
|
||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
// Legacy entries predate target admission persistence. They cannot
|
||||
// be safely attributed, so retain the old live-config fallback.
|
||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
MrfOpKind::Metadata => {
|
||||
let opts = ObjectOptions {
|
||||
version_id: entry.version_id.map(|u| u.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||
Ok(oi) => oi,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
error = %e,
|
||||
"MRF metadata recovery: source object lookup failed"
|
||||
);
|
||||
if should_retry_mrf_source_lookup(&e) {
|
||||
retry_entries.push(entry.clone());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.target_arns.is_empty() {
|
||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
||||
} else {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||
ReplicationQueueAdmission::Queued
|
||||
} else {
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
@@ -1484,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
||||
Ok(retained) => retained,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
Err(read_error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %read_error,
|
||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||
);
|
||||
entries.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Proxy-target selection for reads of objects not yet replicated locally
|
||||
//! (MinIO `getProxyTargets`, bucket-replication.go).
|
||||
//!
|
||||
//! During the active-active replication lag window a GET/HEAD/Tagging request
|
||||
//! for an object the local site does not have yet may be served by proxying to
|
||||
//! a replication target. This module only *selects* the candidate targets; the
|
||||
//! request-path callers perform the remote calls and response translation.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_object_config::get_replication_config;
|
||||
use super::replication_storage_boundary::ObjectOptions;
|
||||
use super::replication_target_boundary::{ReplicationTargetStore, TargetClient};
|
||||
|
||||
/// Returns the replication-target clients eligible to serve a proxied read of
|
||||
/// `bucket/object`, in rule order. Mirrors MinIO's `getProxyTargets`:
|
||||
///
|
||||
/// - the `source-proxy-request` header family was present at all
|
||||
/// (`opts.proxy_request` / `opts.proxy_header_set`, MinIO `ProxyRequest` /
|
||||
/// `ProxyHeaderSet`) -> empty. "true" is the anti-loop marker of an
|
||||
/// already-proxied client read; "false" is what a peer's replication
|
||||
/// worker sends on convergence HEADs so the receiver answers locally —
|
||||
/// proxying that miss back would echo the source object and fake
|
||||
/// convergence, permanently skipping replication;
|
||||
/// - the bucket's versioning is suspended for the object -> empty;
|
||||
/// - no replication configuration / no matching rule -> empty;
|
||||
/// - otherwise every distinct target ARN whose rules match the object,
|
||||
/// resolved through the bucket target system, skipping targets that opted
|
||||
/// out of proxying (`disable_proxy`).
|
||||
pub async fn get_proxy_targets(bucket: &str, object: &str, opts: &ObjectOptions) -> Vec<Arc<TargetClient>> {
|
||||
if opts.proxy_request || opts.proxy_header_set {
|
||||
return Vec::new();
|
||||
}
|
||||
if opts.version_suspended {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let cfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(cfg)) => cfg,
|
||||
Ok(None) => return Vec::new(),
|
||||
Err(err) => {
|
||||
debug!(bucket, object, error = %err, "read proxy: failed to load replication config; not proxying");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let arns = cfg.filter_target_arns(&ObjectOpts {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut targets = Vec::with_capacity(arns.len());
|
||||
for arn in arns {
|
||||
let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
debug!(bucket, object, arn, "read proxy: no client for replication target ARN");
|
||||
continue;
|
||||
};
|
||||
if client.disable_proxy {
|
||||
continue;
|
||||
}
|
||||
targets.push(client);
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn opts() -> ObjectOptions {
|
||||
ObjectOptions::default()
|
||||
}
|
||||
|
||||
/// Anti-loop: a request that was already proxied by a peer must never be
|
||||
/// proxied onward, regardless of replication configuration.
|
||||
#[tokio::test]
|
||||
async fn proxy_request_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_request: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// MinIO `ProxyHeaderSet` parity: the header family being present at all
|
||||
/// disables proxying, even with the value "false" — that is what a
|
||||
/// peer's replication worker sends on convergence HEADs.
|
||||
#[tokio::test]
|
||||
async fn proxy_header_set_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
proxy_header_set: true,
|
||||
proxy_request: false,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// Suspended versioning disables proxying (MinIO parity): the local null
|
||||
/// version is authoritative and a remote read could resurrect data.
|
||||
#[tokio::test]
|
||||
async fn version_suspended_yields_no_targets() {
|
||||
let targets = get_proxy_targets(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..opts()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
|
||||
/// A bucket without replication configuration has nothing to proxy to.
|
||||
/// (No metadata system is running in unit tests, so the config lookup
|
||||
/// resolves to "no configuration" — the same empty-result contract.)
|
||||
#[tokio::test]
|
||||
async fn missing_replication_config_yields_no_targets() {
|
||||
let targets = get_proxy_targets("bucket-without-replication", "object", &opts()).await;
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,15 @@
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
use super::replication_filemeta_boundary::MrfReplicateEntry;
|
||||
|
||||
/// Kept test-only: the runtime consumer was the worker HEAD's fake proxy
|
||||
/// counting (removed in backlog#1675 P1-5); the resyncer tests still pin the
|
||||
/// classifier's semantics for the real client read-proxy failure accounting.
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::should_count_head_proxy_failure;
|
||||
pub use rustfs_replication::{BucketReplicationResyncStatus, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus};
|
||||
pub(crate) use rustfs_replication::{
|
||||
is_version_id_mismatch, resync_state_accepts_update, sanitize_resync_error_detail, should_auto_resume_resync,
|
||||
should_count_head_proxy_failure,
|
||||
is_version_id_mismatch, resync_state_accepts_update, resync_status_duration, sanitize_resync_error_detail,
|
||||
should_auto_resume_resync,
|
||||
};
|
||||
|
||||
#[allow(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1161,6 +1161,31 @@ mod tests {
|
||||
assert!(all.contains_key("proxy-only-bucket"));
|
||||
}
|
||||
|
||||
/// Pins the read-proxy metric contract (backlog#1675 P1-5): the API
|
||||
/// strings the GET/HEAD/Tagging proxy paths record map onto the
|
||||
/// get/head/tagging totals, and only unexpected failures raise the
|
||||
/// failed counters.
|
||||
#[tokio::test]
|
||||
async fn test_proxy_stats_map_read_proxy_apis_to_totals() {
|
||||
let stats = ReplicationStats::new();
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObject", true).await;
|
||||
stats.inc_proxy("proxy-bucket", "HeadObject", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "GetObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "PutObjectTagging", false).await;
|
||||
stats.inc_proxy("proxy-bucket", "DeleteObjectTagging", true).await;
|
||||
|
||||
let metric = stats.get_proxy_stats("proxy-bucket").await;
|
||||
assert_eq!(metric.get_total, 2);
|
||||
assert_eq!(metric.get_failed, 1);
|
||||
assert_eq!(metric.head_total, 1);
|
||||
assert_eq!(metric.head_failed, 0);
|
||||
assert_eq!(metric.get_tag_total, 1);
|
||||
assert_eq!(metric.put_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_total, 1);
|
||||
assert_eq!(metric.delete_tag_failed, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_calculate_bucket_replication_stats_merges_resync_metrics() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
@@ -36,11 +36,15 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
pub use rustfs_replication::SsecPassthroughCapability;
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
};
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -65,6 +69,8 @@ static STANDARD_HEADERS: &[&str] = &[
|
||||
];
|
||||
|
||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||
pub(crate) const ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED: &str = "replication target does not support SSE-C passthrough: the replica would lose its decryption material \
|
||||
(run ?replication-check to re-probe)";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplicationSourceEncryption {
|
||||
@@ -146,6 +152,13 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||
/// customer-algorithm header and let the crate-owned policy decide.
|
||||
pub(crate) fn ssec_passthrough_evidence_present(head: &HeadObjectOutput) -> bool {
|
||||
rustfs_replication::ssec_passthrough_evidence_present(head.sse_customer_algorithm.as_deref())
|
||||
}
|
||||
|
||||
pub(crate) struct ReplicationTargetStore;
|
||||
|
||||
impl ReplicationTargetStore {
|
||||
@@ -165,6 +178,17 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
/// Returns the cached verdict and whether it has outlived its TTL.
|
||||
pub(crate) async fn ssec_passthrough_capability(arn: &str) -> (SsecPassthroughCapability, bool) {
|
||||
BucketTargetSys::get().ssec_passthrough_capability(arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn record_ssec_passthrough_capability(arn: &str, capability: SsecPassthroughCapability) {
|
||||
BucketTargetSys::get()
|
||||
.record_ssec_passthrough_capability(arn, capability)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
@@ -898,6 +922,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Pins the HeadObjectOutput field extraction feeding the crate-owned
|
||||
/// evidence judgment (the gate/evidence policy matrix itself is pinned in
|
||||
/// `rustfs-replication`'s object tests).
|
||||
#[test]
|
||||
fn ssec_passthrough_evidence_requires_customer_algorithm_echo() {
|
||||
let with_evidence = HeadObjectOutput::builder().sse_customer_algorithm("AES256").build();
|
||||
assert!(ssec_passthrough_evidence_present(&with_evidence));
|
||||
|
||||
let empty_algorithm = HeadObjectOutput::builder().sse_customer_algorithm("").build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&empty_algorithm),
|
||||
"an empty echo is not evidence of preserved SSE-C material"
|
||||
);
|
||||
|
||||
let without_evidence = HeadObjectOutput::builder().e_tag("\"abc\"").content_length(8).build();
|
||||
assert!(
|
||||
!ssec_passthrough_evidence_present(&without_evidence),
|
||||
"a plain HEAD response must classify the target as having dropped the material"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_adds_ssec_checksum_metadata() {
|
||||
let metadata = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||
|
||||
@@ -95,7 +95,6 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GetRequest {
|
||||
pub buffer: Vec<u8>,
|
||||
pub offset: i64,
|
||||
@@ -107,11 +106,12 @@ pub struct GetRequest {
|
||||
pub setting_object_info: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct GetResponse {
|
||||
pub size: i64,
|
||||
//pub error: error,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub did_read: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
pub object_info: ObjectInfo,
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
@@ -27,7 +28,6 @@ use tracing::warn;
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedGetOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub is_replication_ready_for_delete_marker: bool,
|
||||
@@ -77,7 +77,7 @@ impl GetObjectOptions {
|
||||
}
|
||||
}
|
||||
if self.checksum {
|
||||
headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED"));
|
||||
headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED"));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
@@ -360,7 +360,6 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
with_versions: bool,
|
||||
|
||||
@@ -137,8 +137,8 @@ impl Default for PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
|
||||
@@ -149,6 +149,7 @@ impl PutObjectOptions {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn set_match_etag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
|
||||
@@ -259,6 +260,7 @@ impl PutObjectOptions {
|
||||
header
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
|
||||
@@ -55,7 +55,6 @@ pub struct RemoveBucketOptions {
|
||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
pub replication_delete_marker: bool,
|
||||
pub replication_status: ReplicationStatus,
|
||||
@@ -465,10 +464,10 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct RemoveObjectError {
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
object_name: String,
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
version_id: String,
|
||||
err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
@@ -372,8 +372,8 @@ pub struct Checksum {
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl Checksum {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Checksum {
|
||||
@@ -385,7 +385,7 @@ impl Checksum {
|
||||
Checksum::default()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||
let b = match base64_decode(s.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
@@ -412,7 +412,7 @@ impl Checksum {
|
||||
base64_encode(&self.r)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn raw(&self) -> Option<Vec<u8>> {
|
||||
if !self.is_set() {
|
||||
return None;
|
||||
|
||||
@@ -37,16 +37,17 @@ pub struct PutObjReader {
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(reader: HashReader) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
self.reader = enc_reader;
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::{
|
||||
http::headers::{
|
||||
AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1,
|
||||
AMZ_CHECKSUM_SHA256,
|
||||
},
|
||||
net::get_endpoint_url,
|
||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||
};
|
||||
@@ -1383,12 +1387,12 @@ pub(crate) fn to_object_info_for_provider(
|
||||
};
|
||||
|
||||
// Extract checksums
|
||||
let checksum_crc32 = get_header("x-amz-checksum-crc32");
|
||||
let checksum_crc32c = get_header("x-amz-checksum-crc32c");
|
||||
let checksum_sha1 = get_header("x-amz-checksum-sha1");
|
||||
let checksum_sha256 = get_header("x-amz-checksum-sha256");
|
||||
let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
|
||||
let checksum_mode = get_header("x-amz-checksum-mode");
|
||||
let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32);
|
||||
let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C);
|
||||
let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1);
|
||||
let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256);
|
||||
let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME);
|
||||
let checksum_mode = get_header(AMZ_CHECKSUM_MODE);
|
||||
|
||||
// Build and return the ObjectInfo struct
|
||||
Ok(ObjectInfo {
|
||||
|
||||
@@ -86,6 +86,25 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Error for a peer that reported `success = false` without an `error_info` payload.
|
||||
///
|
||||
/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError`
|
||||
/// instead of `DiskError`. The message names the operation (and the bucket, where the
|
||||
/// operation has one) and nothing else, for two reasons:
|
||||
///
|
||||
/// - `finalize_result` classifies failures by message substring, so any text matching
|
||||
/// `message_has_network_needle` would take an answering peer offline and evict its
|
||||
/// connection over a plain application-level rejection.
|
||||
/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered
|
||||
/// message, so a per-peer detail such as the peer address would split one shared
|
||||
/// failure into single-count buckets and downgrade the dominant error.
|
||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||
match bucket {
|
||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<BucketStats> {
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
@@ -696,7 +715,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("local_storage_info", None));
|
||||
}
|
||||
let data = response.storage_info;
|
||||
|
||||
@@ -719,7 +738,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("server_info", None));
|
||||
}
|
||||
let data = response.server_properties;
|
||||
|
||||
@@ -742,7 +761,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_cpus", None));
|
||||
}
|
||||
let data = response.cpus;
|
||||
|
||||
@@ -765,7 +784,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_net_info", None));
|
||||
}
|
||||
let data = response.net_info;
|
||||
|
||||
@@ -788,7 +807,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_partitions", None));
|
||||
}
|
||||
let data = response.partitions;
|
||||
|
||||
@@ -811,7 +830,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_os_info", None));
|
||||
}
|
||||
let data = response.os_info;
|
||||
|
||||
@@ -832,7 +851,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_se_linux_info", None));
|
||||
}
|
||||
let data = response.sys_services;
|
||||
|
||||
@@ -857,7 +876,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_sys_config", None));
|
||||
}
|
||||
let data = response.sys_config;
|
||||
|
||||
@@ -882,7 +901,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_sys_errors", None));
|
||||
}
|
||||
let data = response.sys_errors;
|
||||
|
||||
@@ -907,7 +926,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_mem_info", None));
|
||||
}
|
||||
let data = response.mem_info;
|
||||
|
||||
@@ -939,7 +958,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_metrics", None));
|
||||
}
|
||||
let data = response.realtime_metrics;
|
||||
|
||||
@@ -964,7 +983,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_live_events", None));
|
||||
}
|
||||
|
||||
Ok(PeerLiveEventsBatch {
|
||||
@@ -989,7 +1008,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("get_proc_info", None));
|
||||
}
|
||||
let data = response.proc_info;
|
||||
|
||||
@@ -1016,7 +1035,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("start_profiling", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1323,7 +1342,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1346,7 +1365,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1369,7 +1388,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("delete_policy", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1392,7 +1411,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_policy", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1417,7 +1436,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_policy_mapping", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1440,7 +1459,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("delete_user", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1463,7 +1482,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("delete_service_account", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1487,7 +1506,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_user", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1510,7 +1529,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_service_account", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1533,7 +1552,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_group", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1554,7 +1573,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("reload_site_replication_config", None));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1597,7 +1616,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("signal_service", None));
|
||||
}
|
||||
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
||||
Ok(response)
|
||||
@@ -1667,7 +1686,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("reload_pool_meta", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1691,7 +1710,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("stop_rebalance", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1725,7 +1744,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("load_rebalance_meta", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1753,7 +1772,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("start_decommission", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1777,7 +1796,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("decommission_cancel", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1801,7 +1820,7 @@ impl PeerRestClient {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(Error::other(""));
|
||||
return Err(peer_failure_without_details("clear_decommission", None));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1947,6 +1966,8 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error_reduce::reduce_errs;
|
||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||
use serde_json::Value;
|
||||
@@ -3098,4 +3119,115 @@ mod tests {
|
||||
&& span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest")
|
||||
}));
|
||||
}
|
||||
|
||||
/// Every operation name passed to `peer_failure_without_details` in this file.
|
||||
const PEER_FAILURE_OPS: &[&str] = &[
|
||||
"local_storage_info",
|
||||
"server_info",
|
||||
"get_cpus",
|
||||
"get_net_info",
|
||||
"get_partitions",
|
||||
"get_os_info",
|
||||
"get_se_linux_info",
|
||||
"get_sys_config",
|
||||
"get_sys_errors",
|
||||
"get_mem_info",
|
||||
"get_metrics",
|
||||
"get_live_events",
|
||||
"get_proc_info",
|
||||
"start_profiling",
|
||||
"load_bucket_metadata",
|
||||
"delete_bucket_metadata",
|
||||
"delete_policy",
|
||||
"load_policy",
|
||||
"load_policy_mapping",
|
||||
"delete_user",
|
||||
"delete_service_account",
|
||||
"load_user",
|
||||
"load_service_account",
|
||||
"load_group",
|
||||
"reload_site_replication_config",
|
||||
"signal_service",
|
||||
"reload_pool_meta",
|
||||
"stop_rebalance",
|
||||
"load_rebalance_meta",
|
||||
"start_decommission",
|
||||
"decommission_cancel",
|
||||
"clear_decommission",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||
for op in PEER_FAILURE_OPS {
|
||||
let message = peer_failure_without_details(op, None).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
}
|
||||
|
||||
for op in ["load_bucket_metadata", "delete_bucket_metadata"] {
|
||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||
// same operation must stay a single dominant error instead of one bucket per peer.
|
||||
let per_peer_errs = (0..4)
|
||||
.map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))))
|
||||
.collect::<Vec<_>>();
|
||||
let (count, dominant) = reduce_errs(&per_peer_errs, &[]);
|
||||
assert_eq!(count, 4, "one shared failure must not split into per-peer buckets");
|
||||
assert_eq!(
|
||||
dominant,
|
||||
Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(),
|
||||
peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string()
|
||||
);
|
||||
assert_ne!(
|
||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(),
|
||||
peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_never_reads_as_a_network_failure() {
|
||||
// `finalize_result` marks the peer offline and evicts its connection whenever the
|
||||
// message matches a network needle. A peer that answered `success = false` is alive,
|
||||
// so no operation or bucket name may push this text over that classifier.
|
||||
for op in PEER_FAILURE_OPS {
|
||||
let err = peer_failure_without_details(op, None);
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&err),
|
||||
"{op} must not read as a transport failure: {err}"
|
||||
);
|
||||
|
||||
let scoped = peer_failure_without_details(op, Some("bucket-name"));
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&scoped),
|
||||
"{op} must not read as a transport failure: {scoped}"
|
||||
);
|
||||
}
|
||||
|
||||
// The bucket name is caller-supplied. Every needle carries a space, which S3 bucket
|
||||
// names cannot, and the name is closed by `)` before the literal text resumes, so no
|
||||
// needle can straddle the boundary either.
|
||||
for bucket in [
|
||||
"timed-out",
|
||||
"connection-reset",
|
||||
"transport-error",
|
||||
"broken-pipe",
|
||||
"unavailable-logs",
|
||||
] {
|
||||
let err = peer_failure_without_details("load_bucket_metadata", Some(bucket));
|
||||
assert!(
|
||||
!PeerRestClient::is_network_like_error(&err),
|
||||
"bucket {bucket} must not push the message over the network classifier: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,6 +214,21 @@ fn pool_write_quorum(participant_count: usize) -> usize {
|
||||
(participant_count / 2) + 1
|
||||
}
|
||||
|
||||
/// Error for a peer that reported `success = false` without an error payload.
|
||||
///
|
||||
/// The message must stay identical across the peers of one operation: `reduce_errs`
|
||||
/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address,
|
||||
/// timing) would split one shared failure into single-count buckets and downgrade a real
|
||||
/// dominant error into `ErasureWriteQuorum`.
|
||||
///
|
||||
/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape.
|
||||
fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error {
|
||||
match bucket {
|
||||
Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")),
|
||||
None => Error::other(format!("{op}: peer returned failure without error details")),
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Error> {
|
||||
if per_pool_errs.is_empty() {
|
||||
return Some(Error::ErasureWriteQuorum);
|
||||
@@ -1078,7 +1093,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("heal_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1105,7 +1120,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("list_bucket", None))
|
||||
};
|
||||
}
|
||||
let bucket_infos = response
|
||||
@@ -1136,9 +1151,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
"make_bucket({bucket}): peer returned failure without error details"
|
||||
)))
|
||||
Err(peer_failure_without_details("make_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1162,7 +1175,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("get_bucket_info", Some(bucket)))
|
||||
};
|
||||
}
|
||||
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
|
||||
@@ -1190,7 +1203,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
return if let Some(err) = response.error {
|
||||
Err(err.into())
|
||||
} else {
|
||||
Err(Error::other(""))
|
||||
Err(peer_failure_without_details("delete_bucket", Some(bucket)))
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2314,4 +2327,37 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_names_operation_and_bucket() {
|
||||
for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] {
|
||||
let message = peer_failure_without_details(op, Some("ops-bucket")).to_string();
|
||||
assert!(message.contains(op), "{op} message must name the operation: {message}");
|
||||
assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}");
|
||||
}
|
||||
|
||||
let message = peer_failure_without_details("list_bucket", None).to_string();
|
||||
assert!(message.contains("list_bucket"), "cluster-wide message must name the operation");
|
||||
assert!(!message.trim().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() {
|
||||
// reduce_errs groups Io errors by kind plus rendered message: peers failing the
|
||||
// same operation on the same bucket must still reach quorum as one dominant error.
|
||||
let per_pool_errs = vec![
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared"))),
|
||||
];
|
||||
assert_eq!(
|
||||
reduce_pool_write_quorum_errs(&per_pool_errs),
|
||||
Some(peer_failure_without_details("delete_bucket", Some("shared")))
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
peer_failure_without_details("delete_bucket", Some("shared")),
|
||||
peer_failure_without_details("get_bucket_info", Some("shared"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ use rustfs_config::{
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit webhook settings.
|
||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit MQTT settings.
|
||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
|
||||
@@ -12,12 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![KV {
|
||||
@@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
hidden_if_empty: false,
|
||||
}])
|
||||
});
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Config {
|
||||
pub bitrot: String,
|
||||
pub sleep: Duration,
|
||||
pub io_count: usize,
|
||||
pub drive_workers: usize,
|
||||
pub cache: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn bitrot_scan_cycle(&self) -> Duration {
|
||||
self.cache
|
||||
}
|
||||
|
||||
pub fn get_workers(&self) -> usize {
|
||||
self.drive_workers
|
||||
}
|
||||
|
||||
pub fn update(&mut self, nopts: &Config) {
|
||||
self.bitrot = nopts.bitrot.clone();
|
||||
self.io_count = nopts.io_count;
|
||||
self.sleep = nopts.sleep;
|
||||
self.drive_workers = nopts.drive_workers;
|
||||
}
|
||||
}
|
||||
|
||||
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
||||
|
||||
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
||||
match parse_bool(s) {
|
||||
Ok(enabled) => {
|
||||
if enabled {
|
||||
Ok(Duration::from_secs_f64(0.0))
|
||||
} else {
|
||||
Ok(Duration::from_secs_f64(-1.0))
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
if !s.ends_with("m") {
|
||||
return Err(Error::other("unknown format"));
|
||||
}
|
||||
|
||||
match s.trim_end_matches('m').parse::<u64>() {
|
||||
Ok(months) => {
|
||||
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
||||
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
||||
}
|
||||
|
||||
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
||||
}
|
||||
Err(err) => Err(Error::other(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
mod audit;
|
||||
pub mod com;
|
||||
#[allow(dead_code)]
|
||||
pub mod heal;
|
||||
mod notify;
|
||||
mod oidc;
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::bucket::{
|
||||
lifecycle::{
|
||||
LifecycleExpiryConfigs,
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||
@@ -1996,11 +1997,11 @@ impl PoolMeta {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||
struct PoolInfo {
|
||||
position: usize,
|
||||
completed: bool,
|
||||
#[allow(dead_code, reason = "written but never read back (backlog#1823)")]
|
||||
decom_started: bool,
|
||||
}
|
||||
|
||||
@@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
|
||||
action.delete()
|
||||
}
|
||||
|
||||
fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
|
||||
if !apply_actions || applied {
|
||||
return Ok(true);
|
||||
@@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
configs: LifecycleExpiryConfigs,
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
if bucket == RUSTFS_META_BUCKET {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let configs = get_expiry_configs(self, bucket).await?;
|
||||
if configs.lifecycle.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(HealLifecycleExpiryContext { configs }))
|
||||
}
|
||||
|
||||
pub async fn enqueue_heal_lifecycle_expiry(
|
||||
self: &Arc<Self>,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&crate::object_api::ObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let object_info = if let Some(object_info) = object_info {
|
||||
if object_info.bucket != bucket || object_info.name != object {
|
||||
return Ok(false);
|
||||
}
|
||||
let snapshot_version_id = object_info
|
||||
.version_id
|
||||
.filter(|version_id| !version_id.is_nil())
|
||||
.map(|version_id| version_id.to_string());
|
||||
if snapshot_version_id.as_deref() != version_id {
|
||||
return Ok(false);
|
||||
}
|
||||
object_info.clone()
|
||||
} else {
|
||||
match self
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.map(str::to_string),
|
||||
versioned: version_id.is_some(),
|
||||
expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(object_info) => object_info,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
};
|
||||
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
|
||||
if !lifecycle_action_skips_heal_version(event.action) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
|
||||
}
|
||||
|
||||
async fn save_current_pool_meta(&self) -> Result<()> {
|
||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let snapshot = {
|
||||
@@ -4287,6 +4365,19 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_action_skips_heal_version_for_every_delete_action() {
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
|
||||
assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
|
||||
assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
|
||||
let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
|
||||
@@ -4958,13 +5049,19 @@ fn is_disk_online_state(state: &str) -> bool {
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_total_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
|
||||
)]
|
||||
fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
|
||||
fallback_free_capacity_dedup(disks)
|
||||
}
|
||||
|
||||
@@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the pool/set layers
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
self.get_disks_for_heal_object(object, opts)?
|
||||
.check_abandoned_parts(bucket, object, opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1996,7 +1996,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() {
|
||||
async fn sets_check_abandoned_parts_rejects_invalid_set_scope() {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let sets = Sets {
|
||||
id: format.id,
|
||||
@@ -2021,10 +2021,21 @@ mod tests {
|
||||
};
|
||||
|
||||
let err = sets
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("abandoned-parts ownership should stay above the pool/set storage layers");
|
||||
assert!(matches!(err, StorageError::NotImplemented));
|
||||
.expect_err("out-of-range abandoned-parts set scope must fail closed");
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "set" && reason.contains("invalid heal set index 1")),
|
||||
"unexpected invalid set error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks,
|
||||
|
||||
@@ -6562,7 +6562,7 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn get_metrics(&self) -> DiskMetrics {
|
||||
DiskMetrics::default()
|
||||
}
|
||||
|
||||
@@ -820,10 +820,263 @@ impl BitrotWriterWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
|
||||
//
|
||||
// A broken hash implementation (bad SIMD feature combination, platform drift, a
|
||||
// key-handling regression) fails silently: every shard reads back "corrupt",
|
||||
// heal rewrites data that was fine, and cross-platform clusters disagree about
|
||||
// which copy is healthy. The self-test below pins the algorithms the moment a
|
||||
// process starts, so a drifted build announces itself instead of quietly
|
||||
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
|
||||
// analysis-2026-08-16.md §6 HS-11.
|
||||
|
||||
/// Length of the deterministic self-test payload.
|
||||
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
|
||||
|
||||
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
|
||||
/// (the production default). Pinned so any platform or build where the
|
||||
/// implementation drifts fails startup instead of miss-hashing shards.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
|
||||
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
|
||||
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
|
||||
];
|
||||
|
||||
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
|
||||
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
|
||||
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
|
||||
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
|
||||
];
|
||||
|
||||
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
|
||||
/// Highway digests above this one is externally verifiable, so it guards the
|
||||
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
|
||||
/// self-computed constants.
|
||||
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
|
||||
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
|
||||
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
|
||||
];
|
||||
|
||||
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
|
||||
/// platform and every run hashes the same 4096 bytes.
|
||||
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
|
||||
let mut state = 0x9E37_79B9_7F4A_7C15u64;
|
||||
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
|
||||
for byte in payload.iter_mut() {
|
||||
state ^= state >> 12;
|
||||
state ^= state << 25;
|
||||
state ^= state >> 27;
|
||||
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
|
||||
}
|
||||
payload
|
||||
}
|
||||
|
||||
/// Why a bitrot self-test failed.
|
||||
#[derive(Debug)]
|
||||
pub enum BitrotSelfTestError {
|
||||
/// A known-answer digest mismatched the pinned constant.
|
||||
KnownAnswerMismatch {
|
||||
algorithm: &'static str,
|
||||
got: String,
|
||||
want: String,
|
||||
},
|
||||
/// A freshly encoded shard failed `bitrot_verify`.
|
||||
RoundtripVerify { algorithm: &'static str, detail: String },
|
||||
/// A verified roundtrip read back different bytes than were written.
|
||||
RoundtripReadback { algorithm: &'static str },
|
||||
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
|
||||
TamperNotRejected {
|
||||
algorithm: &'static str,
|
||||
tampered: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BitrotSelfTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::KnownAnswerMismatch { algorithm, got, want } => {
|
||||
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
|
||||
}
|
||||
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
|
||||
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
|
||||
Self::TamperNotRejected { algorithm, tampered } => {
|
||||
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BitrotSelfTestError {}
|
||||
|
||||
fn self_test_hex(bytes: &[u8]) -> String {
|
||||
rustfs_utils::hex(bytes)
|
||||
}
|
||||
|
||||
// (kept as a named one-liner so every KAT failure site reads the same; the
|
||||
// underlying formatter is the shared `rustfs_utils::hex`)
|
||||
|
||||
/// Compare a digest against its pinned constant. Split out so a test can drive
|
||||
/// it with a wrong constant and prove the mismatch path fires.
|
||||
fn bitrot_kat_check(
|
||||
algorithm: &'static str,
|
||||
algo: &HashAlgorithm,
|
||||
payload: &[u8],
|
||||
expected: &[u8; 32],
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let digest = algo.hash_encode(payload);
|
||||
let digest = digest.as_ref();
|
||||
if digest.len() != expected.len() || digest != expected.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm,
|
||||
got: self_test_hex(digest),
|
||||
want: self_test_hex(expected),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
|
||||
/// every block back through `BitrotReader` comparing bytes.
|
||||
async fn bitrot_roundtrip_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encode failed: {err}"),
|
||||
})?;
|
||||
}
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
if encoded.len() != on_disk {
|
||||
return Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
|
||||
});
|
||||
}
|
||||
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: err.to_string(),
|
||||
})?;
|
||||
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
|
||||
let mut offset = 0usize;
|
||||
while offset < payload.len() {
|
||||
let want = shard_size.min(payload.len() - offset);
|
||||
let mut buf = vec![0u8; want];
|
||||
let read = reader
|
||||
.read(&mut buf)
|
||||
.await
|
||||
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("read back failed at offset {offset}: {err}"),
|
||||
})?;
|
||||
if read != want || buf[..read] != payload[offset..offset + read] {
|
||||
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
|
||||
}
|
||||
offset += read;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flip one byte and require `bitrot_verify` to reject the result.
|
||||
async fn bitrot_tamper_check(
|
||||
algorithm: &'static str,
|
||||
algo: HashAlgorithm,
|
||||
payload: &[u8],
|
||||
shard_size: usize,
|
||||
tampered: &'static str,
|
||||
flip_at: usize,
|
||||
) -> Result<(), BitrotSelfTestError> {
|
||||
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("self-test encode should not fail");
|
||||
}
|
||||
let mut corrupt = writer.into_inner().into_inner();
|
||||
let flip_index = flip_at % corrupt.len();
|
||||
corrupt[flip_index] ^= 0x80;
|
||||
|
||||
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
|
||||
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
|
||||
// The flipped byte must be rejected as a hash mismatch specifically, not
|
||||
// by any incidental read error: an in-memory cursor cannot fail reads,
|
||||
// so accepting any other failure here would mask a verify path that
|
||||
// errors out before it ever compares hashes.
|
||||
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
|
||||
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
|
||||
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
|
||||
algorithm,
|
||||
detail: format!("tampered shard rejected with an unexpected error: {err}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify every bitrot algorithm this crate can write or verify in production:
|
||||
/// both streaming Highway variants roundtrip end to end (encode → size formula
|
||||
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
|
||||
/// and the leading hash, while all three hashed algorithms reproduce their
|
||||
/// pinned known-answer digests.
|
||||
///
|
||||
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
|
||||
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
|
||||
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
|
||||
let payload = bitrot_self_test_payload();
|
||||
|
||||
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
|
||||
// itself, before any self-pinned constants are consulted.
|
||||
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
|
||||
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
|
||||
return Err(BitrotSelfTestError::KnownAnswerMismatch {
|
||||
algorithm: "SHA256",
|
||||
got: self_test_hex(abc.as_ref()),
|
||||
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
|
||||
});
|
||||
}
|
||||
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256S",
|
||||
&HashAlgorithm::HighwayHash256S,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
|
||||
)?;
|
||||
bitrot_kat_check(
|
||||
"HighwayHash256SLegacy",
|
||||
&HashAlgorithm::HighwayHash256SLegacy,
|
||||
&payload,
|
||||
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
|
||||
)?;
|
||||
|
||||
for (algorithm, algo) in [
|
||||
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
|
||||
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
|
||||
] {
|
||||
// Full blocks plus a partial tail, exactly like a real part stripe.
|
||||
let tail_len = 2 * 1024 + 333;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
|
||||
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
|
||||
// One flipped byte in the final data block, one in the first leading
|
||||
// hash: both must fail verification.
|
||||
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
|
||||
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test,
|
||||
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
|
||||
};
|
||||
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
|
||||
use bytes::Bytes;
|
||||
@@ -1090,6 +1343,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_payload_is_deterministic() {
|
||||
// Two independent builds of the payload must agree byte for byte, or
|
||||
// the pinned known-answer digests below would be meaningless.
|
||||
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
|
||||
let payload = bitrot_self_test_payload();
|
||||
let wrong = [0u8; 32];
|
||||
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
|
||||
.expect_err("a zeroed digest must never match");
|
||||
match err {
|
||||
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
|
||||
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_self_test_passes() {
|
||||
bitrot_self_test()
|
||||
.await
|
||||
.expect("the pinned digests and roundtrip checks must all pass on this platform");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
|
||||
let mut counting = VectoredCountingWriter::default();
|
||||
@@ -1189,7 +1468,7 @@ mod tests {
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
Cursor::new(corrupt),
|
||||
std::io::Cursor::new(corrupt),
|
||||
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
|
||||
data.len(),
|
||||
algo,
|
||||
@@ -1282,7 +1561,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
|
||||
let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
|
||||
let mut out = [0u8; 5];
|
||||
let err = reader
|
||||
.read(&mut out)
|
||||
@@ -1407,7 +1686,7 @@ mod tests {
|
||||
(HashAlgorithm::HighwayHash256, true),
|
||||
] {
|
||||
let label = format!("{algo:?}");
|
||||
let writer = Cursor::new(Vec::<u8>::new());
|
||||
let writer = std::io::Cursor::new(Vec::<u8>::new());
|
||||
let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
|
||||
w.write(&[7u8; 16]).await.unwrap();
|
||||
let written = w.into_inner().into_inner();
|
||||
@@ -1492,7 +1771,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo);
|
||||
w.write(payload).await.unwrap();
|
||||
w.into_inner().into_inner()
|
||||
}
|
||||
@@ -1600,7 +1879,7 @@ mod tests {
|
||||
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
|
||||
for &size in &[1usize, 16, 17, 32, 40, 48] {
|
||||
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
|
||||
let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
w.write(chunk).await.unwrap();
|
||||
}
|
||||
@@ -1674,14 +1953,14 @@ mod tests {
|
||||
w.write(&data).await.expect("write shard");
|
||||
|
||||
let mut via_read = vec![0u8; SHARD];
|
||||
let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
|
||||
.read(&mut via_read)
|
||||
.await
|
||||
.expect("read");
|
||||
|
||||
// A buffer with only capacity — no initialized bytes at all.
|
||||
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut via_append, SHARD)
|
||||
.await
|
||||
.expect("read_appending");
|
||||
@@ -1706,7 +1985,7 @@ mod tests {
|
||||
encoded.truncate(encoded.len() - 1);
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a truncated shard must not succeed");
|
||||
@@ -1732,7 +2011,7 @@ mod tests {
|
||||
encoded[last] ^= 0xff;
|
||||
|
||||
let mut out: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut out, SHARD)
|
||||
.await
|
||||
.expect_err("a corrupt shard must not verify");
|
||||
@@ -1844,7 +2123,7 @@ mod tests {
|
||||
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
|
||||
);
|
||||
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
|
||||
let mut streamed = Cursor::new(encoded.clone());
|
||||
let mut streamed = std::io::Cursor::new(encoded.clone());
|
||||
assert!(
|
||||
ShardSource::try_take_block(&mut streamed, 8).is_none(),
|
||||
"a non-Bytes source must stay on the streaming path"
|
||||
@@ -1872,7 +2151,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
|
||||
BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
|
||||
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false)
|
||||
.read_appending(&mut via_stream, SHARD)
|
||||
.await
|
||||
.expect("streaming read");
|
||||
|
||||
@@ -1116,6 +1116,14 @@ mod tests {
|
||||
assert!(encoder_source.is::<reed_solomon_erasure::Error>());
|
||||
}
|
||||
|
||||
// The lifecycle transition worker relies on this arm alone to suppress the
|
||||
// closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would
|
||||
// silently turn shutdown races back into `error!` log spam.
|
||||
#[test]
|
||||
fn is_network_or_host_down_covers_closed_network_connection() {
|
||||
assert!(is_network_or_host_down("transition failed: use of closed network connection", false));
|
||||
}
|
||||
|
||||
// Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in
|
||||
// every set unreachable) must NOT be classified as "all not found",
|
||||
// otherwise ListObjects silently returns an empty listing and masks a full
|
||||
|
||||
@@ -277,6 +277,20 @@ pub struct ObjectOptions {
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
pub replication_request: bool,
|
||||
/// True when the inbound request carried the
|
||||
/// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
|
||||
/// value "true": the request was already proxied by a replication peer,
|
||||
/// so this server must not proxy a local miss onward (anti-loop,
|
||||
/// MinIO-compatible). The header only disables proxying — it grants no
|
||||
/// capability — so no authorization gate is required to honor it.
|
||||
pub proxy_request: bool,
|
||||
/// True when the `source-proxy-request` header family was present at
|
||||
/// all, regardless of value (MinIO's `ProxyHeaderSet`). A replication
|
||||
/// peer sends `source-proxy-request: false` on its worker convergence
|
||||
/// HEADs precisely so the receiver answers locally instead of proxying
|
||||
/// back — otherwise a proxied 404->200 echo makes the worker believe the
|
||||
/// object already converged and it never replicates it.
|
||||
pub proxy_header_set: bool,
|
||||
/// Source-cluster LWW timestamps carried by an authorized replication
|
||||
/// request; None when the source never modified the category. Only the
|
||||
/// replication-authorized options builders may set these.
|
||||
|
||||
@@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DiskStat {
|
||||
pub total_space: u64,
|
||||
|
||||
@@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{fmt::Display, io};
|
||||
use tracing::info;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier config wire version stamped by the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)"
|
||||
)]
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||
@@ -264,7 +272,6 @@ impl Clone for TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfig {
|
||||
pub(crate) fn clone_with_credentials(&self) -> Self {
|
||||
Self {
|
||||
@@ -284,6 +291,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||
@@ -303,6 +311,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||
@@ -322,6 +331,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||
@@ -341,6 +351,7 @@ impl TierConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||
@@ -457,7 +468,7 @@ impl TierWasabi {
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
access_key: &str,
|
||||
@@ -528,7 +539,7 @@ pub struct TierMinIO {
|
||||
}
|
||||
|
||||
impl TierMinIO {
|
||||
#[allow(dead_code)]
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn create<F>(
|
||||
name: &str,
|
||||
endpoint: &str,
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TierConfigMgr {
|
||||
pub fn msg_size(&self) -> usize {
|
||||
100
|
||||
|
||||
@@ -4860,6 +4860,14 @@ impl SetDisks {
|
||||
/// is best-effort maintenance: individual delete failures are logged and
|
||||
/// skipped rather than propagated.
|
||||
pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, false).await
|
||||
}
|
||||
|
||||
pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result<usize> {
|
||||
self.reclaim_orphan_data_dirs_inner(bucket, object, true).await
|
||||
}
|
||||
|
||||
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||
@@ -4967,6 +4975,20 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
if dry_run {
|
||||
removed += 1;
|
||||
debug!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
event = "heal_abandoned_parts",
|
||||
component = "ecstore",
|
||||
subsystem = "heal",
|
||||
state = "dry_run_matched",
|
||||
result = "matched",
|
||||
bucket, object, data_dir = %dir,
|
||||
"Heal abandoned parts dry-run matched orphaned data directory"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match disk
|
||||
.delete(
|
||||
bucket,
|
||||
|
||||
@@ -6998,6 +6998,100 @@ mod tests {
|
||||
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
||||
}
|
||||
|
||||
async fn recv_abandoned_parts_trace(
|
||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
state: &str,
|
||||
) -> rustfs_common::trace_bus::TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("abandoned-parts trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if event.kind == rustfs_common::trace_bus::TraceKind::Heal
|
||||
&& event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts
|
||||
&& event.bucket.as_deref() == Some(bucket)
|
||||
&& event.object.as_deref() == Some(object)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected abandoned-parts trace state {state} for {bucket}/{object}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(),
|
||||
rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_abandoned_parts_dry_run_counts_without_deleting() {
|
||||
let mut trace = rustfs_common::trace_bus::subscribe_trace_events();
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
|
||||
let object_dir = dir.path().join("bucket").join("obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
fs::create_dir_all(object_dir.join(live.to_string()))
|
||||
.await
|
||||
.expect("live data dir should be created");
|
||||
fs::create_dir_all(object_dir.join(orphan.to_string()))
|
||||
.await
|
||||
.expect("orphan data dir should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dry-run abandoned-parts check should succeed");
|
||||
let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await;
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true"));
|
||||
assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved");
|
||||
assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir");
|
||||
|
||||
set.check_abandoned_parts(
|
||||
"bucket",
|
||||
"obj",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("abandoned-parts check should reclaim stale data dir");
|
||||
let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await;
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false"));
|
||||
assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1"));
|
||||
|
||||
assert!(
|
||||
object_dir.join(live.to_string()).exists(),
|
||||
"referenced data dir must remain after reclaim"
|
||||
);
|
||||
assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
@@ -12233,11 +12327,18 @@ mod tests {
|
||||
.expect_err("unsupported copy_object_part should return a typed error");
|
||||
assert!(matches!(copy_part_err, StorageError::NotImplemented));
|
||||
|
||||
let abandoned_err = set_disks
|
||||
.check_abandoned_parts("bucket", "object", &HealOpts::default())
|
||||
set_disks
|
||||
.check_abandoned_parts(
|
||||
"bucket",
|
||||
"object",
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("abandoned-parts check should stay in the upper reconciliation layer");
|
||||
assert!(matches!(abandoned_err, StorageError::NotImplemented));
|
||||
.expect("abandoned-parts check should be callable on empty disk sets");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,6 +16,7 @@ use super::super::*;
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -2057,11 +2058,61 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
Err(Error::DiskNotFound)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
|
||||
// Multipart orphan reconciliation is intentionally retained above the set layer
|
||||
// until there is a concrete caller and a stable lower-level contract to implement.
|
||||
Err(StorageError::NotImplemented)
|
||||
#[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))]
|
||||
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let _write_lock_guard = if !opts.no_lock {
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let removed = if opts.dry_run {
|
||||
self.dry_run_reclaim_orphan_data_dirs(bucket, object).await?
|
||||
} else {
|
||||
self.reclaim_orphan_data_dirs(bucket, object).await?
|
||||
};
|
||||
let state = if opts.dry_run && removed > 0 {
|
||||
"dry_run_matched"
|
||||
} else if removed > 0 {
|
||||
"reclaimed"
|
||||
} else {
|
||||
"checked"
|
||||
};
|
||||
let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX);
|
||||
|
||||
trace_emit(|| {
|
||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts)
|
||||
.with_bucket(bucket)
|
||||
.with_object(object)
|
||||
.with_duration(started_at.elapsed())
|
||||
.with_attr("state", state)
|
||||
.with_attr("dry_run", opts.dry_run)
|
||||
.with_attr("data_dirs", data_dirs)
|
||||
});
|
||||
|
||||
if removed > 0 {
|
||||
trace!(
|
||||
event = "heal_abandoned_parts",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" },
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
data_dirs = removed,
|
||||
"Heal abandoned parts checked object data directories"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3246,4 +3297,223 @@ mod heal_result_report_tests {
|
||||
assert!(result.detail.contains("part 1"));
|
||||
assert!(result.detail.contains("bitrot_failure=true"));
|
||||
}
|
||||
|
||||
// HS-12 (backlog#1874): a versioned DELETE racing an object heal must never
|
||||
// resurrect the deleted version. The heal has real reconstruction work (a
|
||||
// shard of the doomed version is removed), so both sides touch the same
|
||||
// (bucket, object, data_dir); whichever order the ns write lock serializes
|
||||
// them in, the committed delete must win.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_racing_version_delete_never_resurrects_the_deleted_version() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "heal-race-delete-no-resurrect";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned bucket should be created");
|
||||
|
||||
let mut first_reader = PutObjReader::from_vec(vec![0x11; 1024 * 1024]);
|
||||
let first_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut first_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("first version should be written");
|
||||
let first_version = first_info
|
||||
.version_id
|
||||
.expect("versioned put should return the first version id")
|
||||
.to_string();
|
||||
|
||||
let mut second_reader = PutObjReader::from_vec(vec![0x22; 1024 * 1024]);
|
||||
let second_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut second_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("second version should be written");
|
||||
let second_version = second_info
|
||||
.version_id
|
||||
.expect("versioned put should return the second version id")
|
||||
.to_string();
|
||||
|
||||
// Damage one shard of the doomed version so the racing heal performs an
|
||||
// actual reconstruction over its data dir instead of an early exit.
|
||||
let doomed_source = disks[0]
|
||||
.read_version("", bucket, object, &first_version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("doomed version metadata should be readable");
|
||||
let doomed_data_dir = doomed_source
|
||||
.data_dir
|
||||
.expect("non-inline version should have a data directory");
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(doomed_data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("shard damage should be injected before the race");
|
||||
|
||||
let delete_set = set.clone();
|
||||
let (delete_res, heal_res) = tokio::join!(
|
||||
async {
|
||||
delete_set
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(first_version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
},
|
||||
async {
|
||||
set.heal_object(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
},
|
||||
);
|
||||
delete_res.expect("version delete must succeed under lock serialization");
|
||||
// The heal may legitimately report a transient failure when the version
|
||||
// it was rebuilding disappears mid-flight; only the end state matters.
|
||||
drop(heal_res);
|
||||
|
||||
let resurrected = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(first_version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&resurrected, Err(Error::FileVersionNotFound) | Err(Error::ObjectNotFound(..))),
|
||||
"a racing heal must not resurrect the deleted version: {resurrected:?}"
|
||||
);
|
||||
|
||||
let survivor = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(second_version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version must remain readable after the race");
|
||||
assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact");
|
||||
}
|
||||
|
||||
// HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on
|
||||
// the same object. The overwrite's post-commit tail deletes the replaced
|
||||
// data dir without the ns lock (object.rs commit tail), which is exactly
|
||||
// the intersection the audit flagged: the heal must tolerate the tail race
|
||||
// (retryable outcome) and every committed overwrite must survive — the
|
||||
// final current version is exactly the last payload written.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "heal-race-put-overwrite";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
const ROUNDS: usize = 8;
|
||||
const PAYLOAD_SIZE: usize = 256 * 1024;
|
||||
let mut last_etag = String::new();
|
||||
for round in 0..ROUNDS {
|
||||
// Give the heal something to rebuild on alternating rounds: remove a
|
||||
// shard of the current data dir right before the race.
|
||||
if round % 2 == 1 {
|
||||
let current = disks[2]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("current metadata should be readable");
|
||||
if let Some(data_dir) = current.data_dir {
|
||||
let shard = temp_dirs[3]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
if shard.exists() {
|
||||
tokio::fs::remove_file(&shard)
|
||||
.await
|
||||
.expect("shard damage should be injectable mid-race");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let payload = vec![round as u8; PAYLOAD_SIZE];
|
||||
let mut put_reader = PutObjReader::from_vec(payload);
|
||||
let put_opts = ObjectOptions::default();
|
||||
let heal_opts = HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
};
|
||||
let (put_res, heal_res) = tokio::join!(
|
||||
set.put_object(bucket, object, &mut put_reader, &put_opts),
|
||||
set.heal_object(bucket, object, "", &heal_opts),
|
||||
);
|
||||
let put_info = put_res.expect("overwrite must succeed under lock serialization");
|
||||
last_etag = put_info.etag.clone().unwrap_or_default();
|
||||
// Heal outcome is unconstrained (may hit the tail race and report a
|
||||
// retryable error); the invariant is checked on the end state.
|
||||
drop(heal_res);
|
||||
}
|
||||
|
||||
let final_info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("object must remain readable after the race loop");
|
||||
assert_eq!(
|
||||
final_info.size, PAYLOAD_SIZE as i64,
|
||||
"final current version must be the last committed overwrite"
|
||||
);
|
||||
assert_eq!(
|
||||
final_info.etag.unwrap_or_default(),
|
||||
last_etag,
|
||||
"the racing heal loop must never leave a stale or resurrected current version"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
//! per-version `SetDisks::heal_object`.
|
||||
|
||||
use super::super::*;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
@@ -39,12 +40,16 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
/// it must not gate healing logic — the delete-marker vs data path is chosen
|
||||
/// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is
|
||||
/// normalized (nil/absent UUID => `None`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealWalkVersion {
|
||||
/// object key
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<ObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -63,6 +68,7 @@ struct HealWalkCollector {
|
||||
bucket: String,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
objects: Mutex<Vec<HealWalkObject>>,
|
||||
decode_error: Mutex<Option<DiskError>>,
|
||||
version_total: AtomicUsize,
|
||||
@@ -116,10 +122,25 @@ impl HealWalkCollector {
|
||||
|
||||
let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len());
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
// Normalize: nil/absent version id => None.
|
||||
version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
version_id: version_uuid.map(|u| u.to_string()),
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -173,11 +194,26 @@ impl HealWalkCollector {
|
||||
}
|
||||
};
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string());
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let vid = version_uuid.map(|u| u.to_string());
|
||||
if seen.insert(vid.clone()) {
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
versions.push(HealWalkVersion {
|
||||
name: entry.name.clone(),
|
||||
version_id: vid,
|
||||
mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()),
|
||||
lifecycle_object_info,
|
||||
is_delete_marker: fi.deleted,
|
||||
});
|
||||
}
|
||||
@@ -255,6 +291,7 @@ impl SetDisks {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> disk::error::Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2");
|
||||
|
||||
@@ -264,6 +301,7 @@ impl SetDisks {
|
||||
bucket: bucket.to_string(),
|
||||
batch_objects,
|
||||
version_budget: version_budget.max(1),
|
||||
include_lifecycle_object_info,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -347,6 +385,7 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 2,
|
||||
version_budget: 2,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
decode_error: Mutex::new(None),
|
||||
version_total: AtomicUsize::new(0),
|
||||
@@ -388,6 +427,8 @@ mod tests {
|
||||
HealWalkVersion {
|
||||
name: name.to_string(),
|
||||
version_id: Some(id.to_string()),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: dm,
|
||||
}
|
||||
}
|
||||
@@ -491,6 +532,7 @@ mod tests {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 1000,
|
||||
version_budget: 10_000,
|
||||
include_lifecycle_object_info: false,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
version_total: AtomicUsize::new(0),
|
||||
decode_error: Mutex::new(None),
|
||||
@@ -567,7 +609,7 @@ mod tests {
|
||||
.expect("corrupt test metadata should be written");
|
||||
|
||||
let error = set_disks
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2)
|
||||
.heal_walk_versions_page(bucket, "", None, 2, 2, false)
|
||||
.await
|
||||
.expect_err("semantic metadata corruption must fail the heal disk walk");
|
||||
|
||||
|
||||
@@ -5845,6 +5845,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||
// MRF journal intent: partial-write recovery must survive a restart
|
||||
// (HS-01); the heal request below remains the in-memory fast path.
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
bucket,
|
||||
object,
|
||||
uuid::Uuid::try_parse(version_id).ok(),
|
||||
);
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
|
||||
@@ -1077,6 +1077,15 @@ impl SetDisks {
|
||||
"Recoverable decode error triggered read repair"
|
||||
);
|
||||
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
||||
// MRF journal intent: keeps a durable Urgent ECDecode
|
||||
// request alive across restarts even when the in-memory
|
||||
// read-repair request is dropped or lost (HS-01).
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
bucket,
|
||||
object,
|
||||
fi.version_id,
|
||||
);
|
||||
submit_read_repair_heal(
|
||||
bucket,
|
||||
object,
|
||||
|
||||
@@ -18,6 +18,7 @@ use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
|
||||
@@ -256,13 +257,40 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
|
||||
let _ = (bucket, object, opts);
|
||||
// Stale multipart reconciliation is already owned by the lifecycle-driven
|
||||
// background cleanup path in `bucket_lifecycle_ops.rs`. There is currently
|
||||
// no stable object-heal contract that should fan this request out through
|
||||
// pool/set storage layers, so keep the placeholder explicit at the ECStore
|
||||
// boundary instead of dispatching into lower layers.
|
||||
Err(StorageError::NotImplemented)
|
||||
let object = encode_dir_object(object);
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
futures.push(pool.check_abandoned_parts(bucket, &object, opts));
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
for result in join_all(futures).await {
|
||||
if let Err(err) = result
|
||||
&& first_error.is_none()
|
||||
{
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = first_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
trace!(
|
||||
event = EVENT_HEAL_ABANDONED_PARTS,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = "completed",
|
||||
result = "ok",
|
||||
bucket,
|
||||
object,
|
||||
dry_run = opts.dry_run,
|
||||
"Heal abandoned parts completed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ impl ECStore {
|
||||
forward_to: Option<&str>,
|
||||
batch_objects: usize,
|
||||
version_budget: usize,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealWalkVersion>, Option<String>, bool)> {
|
||||
if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() {
|
||||
return Err(Error::other(format!(
|
||||
@@ -43,7 +44,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
self.pools[pool_idx].disk_set[set_idx]
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget)
|
||||
.heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info)
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
@@ -216,6 +216,16 @@ impl std::fmt::Debug for ECStore {
|
||||
/// These delegate to the process-global statics. No local state — the globals
|
||||
/// remain the single source of truth until the migration is complete.
|
||||
impl ECStore {
|
||||
/// Every erasure set across all pools, pool-major order.
|
||||
///
|
||||
/// Read-only queries that must consult each set's own copy of a
|
||||
/// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate
|
||||
/// this instead of the hash-routed store path, which would always land
|
||||
/// on one set (rustfs/backlog#1872).
|
||||
pub fn all_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
|
||||
self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
|
||||
}
|
||||
|
||||
/// Get server configuration (delegates to global)
|
||||
pub fn get_server_config(&self) -> Option<Config> {
|
||||
runtime_sources::server_config()
|
||||
|
||||
@@ -42,6 +42,9 @@ const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::<u64>() * 8;
|
||||
const FILEINFO_PART_BITMAP_WORDS: usize = MAX_FILEINFO_PARTS.div_ceil(FILEINFO_PART_BITMAP_WORD_BITS);
|
||||
|
||||
// Additional constants from Go version
|
||||
// Intentionally duplicated (S3 wire literal): rustfs-replication and
|
||||
// rustfs-object-data-cache carry their own independent "null" constants so
|
||||
// they stay free of a rustfs-filemeta dependency. Keep all three in sync.
|
||||
pub const NULL_VERSION_ID: &str = "null";
|
||||
// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
|
||||
|
||||
|
||||
@@ -89,6 +89,8 @@ async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
@@ -612,7 +612,8 @@ impl HealChannelProcessor {
|
||||
HealRequestSource::Admin
|
||||
| HealRequestSource::AutoHeal
|
||||
| HealRequestSource::Internal
|
||||
| HealRequestSource::ReadRepair => true,
|
||||
| HealRequestSource::ReadRepair
|
||||
| HealRequestSource::Mrf => true,
|
||||
});
|
||||
|
||||
// Build HealOptions with all available fields
|
||||
@@ -767,6 +768,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> crate::Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
|
||||
@@ -23,13 +23,14 @@ use crate::heal::{
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use metrics::gauge;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
@@ -47,6 +48,21 @@ enum HealObjectOutcome {
|
||||
Failed,
|
||||
}
|
||||
|
||||
fn result_object_size_u64(result: &HealResultItem) -> u64 {
|
||||
u64::try_from(result.object_size).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60;
|
||||
const NANOS_PER_SECOND: i128 = 1_000_000_000;
|
||||
|
||||
fn should_skip_new_version(mod_time_unix_nanos: Option<i128>, started_at_secs: u64) -> bool {
|
||||
let Some(mod_time_unix_nanos) = mod_time_unix_nanos else {
|
||||
return false;
|
||||
};
|
||||
let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS);
|
||||
mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND)
|
||||
}
|
||||
|
||||
struct PageConcurrencyGuard {
|
||||
in_flight: Arc<AtomicUsize>,
|
||||
set_label: String,
|
||||
@@ -492,6 +508,7 @@ impl ErasureSetHealer {
|
||||
&mut skipped_objects,
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -658,6 +675,7 @@ impl ErasureSetHealer {
|
||||
skipped_objects: &mut u64,
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -710,6 +728,7 @@ impl ErasureSetHealer {
|
||||
// The end-of-pass summary reports the full failed/skipped counts.
|
||||
let mut transient_skip_samples_logged = 0_u64;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||
|
||||
// backlog#920: select the per-erasure-set DISK-WALK union enumerator when
|
||||
// the scan is Deep OR the request came from AutoHeal — these are the paths
|
||||
@@ -718,17 +737,25 @@ impl ErasureSetHealer {
|
||||
// which stays the default.
|
||||
let use_disk_walk =
|
||||
matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal);
|
||||
let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?;
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
// Get one page of object versions
|
||||
let (objects, next_token, is_truncated) = if use_disk_walk {
|
||||
self.storage
|
||||
.list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref())
|
||||
.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
"",
|
||||
continuation_token.as_deref(),
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info)
|
||||
.await?
|
||||
};
|
||||
let page_is_empty = objects.is_empty();
|
||||
@@ -736,6 +763,7 @@ impl ErasureSetHealer {
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
// Capture the last version identity of this page for the anti-loop guard.
|
||||
let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone()));
|
||||
@@ -751,6 +779,75 @@ impl ErasureSetHealer {
|
||||
continue;
|
||||
}
|
||||
|
||||
if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_new_versions_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_new_version();
|
||||
progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_new_version",
|
||||
"Erasure set object version skipped because it was written after heal started"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(context) = lifecycle_expiry_context.as_ref()
|
||||
&& self
|
||||
.storage
|
||||
.enqueue_heal_lifecycle_expiry(
|
||||
context,
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
item.lifecycle_object_info.as_ref(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*processed_objects = processed_objects.saturating_add(1);
|
||||
completed_in_page = completed_in_page.saturating_add(1);
|
||||
counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.record_skipped_ilm_expired();
|
||||
progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
object = %item.name,
|
||||
version_id = ?item.version_id,
|
||||
state = "skipped_ilm_expired",
|
||||
"Erasure set object version skipped because lifecycle expiry was queued"
|
||||
);
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
resume_manager
|
||||
.set_current_item(Some(bucket.to_string()), Some(item.name.clone()))
|
||||
.await?;
|
||||
@@ -777,7 +874,7 @@ impl ErasureSetHealer {
|
||||
|
||||
let _permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => return (dedup_key, object_name, version_id, Err(err)),
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))),
|
||||
};
|
||||
|
||||
let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label);
|
||||
@@ -788,7 +885,7 @@ impl ErasureSetHealer {
|
||||
// recorded as skipped-ok rather than failed. The delete-marker
|
||||
// vs data path is chosen internally in ops/heal.rs.
|
||||
let result = if cancel_token.is_cancelled() {
|
||||
Err(Error::TaskCancelled)
|
||||
(0, Err(Error::TaskCancelled))
|
||||
} else {
|
||||
match storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
@@ -797,8 +894,9 @@ impl ErasureSetHealer {
|
||||
Ok((result, None))
|
||||
if target_outcomes_complete(&result, &target_endpoints) =>
|
||||
{
|
||||
let object_size = result_object_size_u64(&result);
|
||||
if !replacement_commit_evidence_required {
|
||||
Ok(true)
|
||||
(object_size, Ok(true))
|
||||
} else {
|
||||
match storage
|
||||
.replacement_targets_have_version(
|
||||
@@ -810,27 +908,42 @@ impl ErasureSetHealer {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => Ok(true),
|
||||
Ok(false) => Err(Error::transient_skip(format!(
|
||||
Ok(true) => (object_size, Ok(true)),
|
||||
Ok(false) => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version"
|
||||
))),
|
||||
Err(err) => Err(Error::transient_skip(format!(
|
||||
)))),
|
||||
Err(err) => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}"
|
||||
))),
|
||||
)))),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
Ok((_result, None)) => Ok(true),
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false),
|
||||
Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => Ok(false),
|
||||
HealObjectOutcome::Transient => Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
},
|
||||
Ok((result, None)) if !target_endpoints.is_empty() => (
|
||||
result_object_size_u64(&result),
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed"
|
||||
))),
|
||||
HealObjectOutcome::Failed => Err(err),
|
||||
),
|
||||
Ok((result, None)) => (result_object_size_u64(&result), Ok(true)),
|
||||
Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => {
|
||||
(result_object_size_u64(&result), Ok(false))
|
||||
}
|
||||
Ok((result, Some(err))) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (object_size, Ok(false)),
|
||||
HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||
}
|
||||
}
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent => (0, Ok(false)),
|
||||
HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!(
|
||||
"Skipped heal for {bucket_name}/{object_name} due to transient error: {err}"
|
||||
)))),
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
},
|
||||
}
|
||||
};
|
||||
@@ -839,11 +952,12 @@ impl ErasureSetHealer {
|
||||
});
|
||||
}
|
||||
|
||||
let mut completed_in_page = 0usize;
|
||||
while let Some((key, object, version_id, result)) = page_tasks.next().await {
|
||||
let (object_size, result) = result;
|
||||
match result {
|
||||
Ok(true) => {
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -861,6 +975,7 @@ impl ErasureSetHealer {
|
||||
Ok(false) => {
|
||||
checkpoint_manager.add_processed_object(key).await?;
|
||||
*successful_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -877,6 +992,7 @@ impl ErasureSetHealer {
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
*skipped_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_skipped_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -893,6 +1009,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
Err(err) => {
|
||||
*failed_objects += 1;
|
||||
bytes_processed = bytes_processed.saturating_add(object_size);
|
||||
checkpoint_manager.add_failed_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
@@ -911,6 +1028,11 @@ impl ErasureSetHealer {
|
||||
|
||||
*processed_objects += 1;
|
||||
completed_in_page += 1;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
|
||||
}
|
||||
|
||||
if completed_in_page.is_multiple_of(100) {
|
||||
checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
|
||||
@@ -964,7 +1086,9 @@ impl ErasureSetHealer {
|
||||
progress.objects_scanned = state.total_objects;
|
||||
progress.objects_healed = state.successful_objects;
|
||||
progress.objects_failed = state.failed_objects;
|
||||
progress.bytes_processed = 0; // set to 0 for now, can be extended later
|
||||
progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
|
||||
progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
|
||||
progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
|
||||
progress.set_current_object(state.current_object.clone());
|
||||
}
|
||||
}
|
||||
@@ -1135,13 +1259,15 @@ mod resume_loop_tests {
|
||||
//! that emits programmable multi-version pages. These exercise the real loop
|
||||
//! logic (cursor seeding, per-version dedup, anti-loop guard, absence
|
||||
//! handling) — not merely a mock's own output.
|
||||
use super::{ErasureSetHealer, target_outcomes_complete};
|
||||
use super::{
|
||||
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
|
||||
};
|
||||
use crate::heal::progress::HealProgress;
|
||||
use crate::heal::resume::{
|
||||
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
|
||||
compose_key,
|
||||
};
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
@@ -1149,7 +1275,7 @@ mod resume_loop_tests {
|
||||
use crate::{Error, Result};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tempfile::TempDir;
|
||||
@@ -1160,10 +1286,37 @@ mod resume_loop_tests {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: delete_marker,
|
||||
}
|
||||
}
|
||||
|
||||
fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: version.map(str::to_string),
|
||||
mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)),
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_version_filter_respects_grace_boundary() {
|
||||
let started_at = 1_700_000_000;
|
||||
|
||||
assert!(!should_skip_new_version(None, started_at));
|
||||
assert!(!should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
assert!(should_skip_new_version(
|
||||
Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)),
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_outcomes_require_each_requested_endpoint_once_and_ok() {
|
||||
let result = HealResultItem {
|
||||
@@ -1246,8 +1399,10 @@ mod resume_loop_tests {
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
lifecycle_expired: Mutex<HashSet<String>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
fail_listing: AtomicBool,
|
||||
}
|
||||
@@ -1274,9 +1429,15 @@ mod resume_loop_tests {
|
||||
.unwrap()
|
||||
.insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string()));
|
||||
}
|
||||
fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) {
|
||||
self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version));
|
||||
}
|
||||
fn calls(&self) -> Vec<(String, Option<String>)> {
|
||||
self.heal_calls.lock().unwrap().clone()
|
||||
}
|
||||
fn list_include_lifecycle_object_info_calls(&self) -> Vec<bool> {
|
||||
self.list_include_lifecycle_object_info.lock().unwrap().clone()
|
||||
}
|
||||
fn fail_listing(&self) {
|
||||
self.fail_listing.store(true, Ordering::SeqCst);
|
||||
}
|
||||
@@ -1330,6 +1491,23 @@ mod resume_loop_tests {
|
||||
async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test))
|
||||
}
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(self
|
||||
.lifecycle_expired
|
||||
.lock()
|
||||
.unwrap()
|
||||
.contains(&compose_key(object, version_id)))
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -1386,7 +1564,12 @@ mod resume_loop_tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_include_lifecycle_object_info
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(include_lifecycle_object_info);
|
||||
if self.fail_listing.load(Ordering::SeqCst) {
|
||||
return Err(Error::other("injected listing failure"));
|
||||
}
|
||||
@@ -1476,6 +1659,7 @@ mod resume_loop_tests {
|
||||
|
||||
/// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result).
|
||||
async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) {
|
||||
let state = env.resume.get_state().await;
|
||||
let mut current_object_index = 0usize;
|
||||
let mut processed = 0u64;
|
||||
let mut successful = 0u64;
|
||||
@@ -1494,6 +1678,7 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result)
|
||||
@@ -1559,6 +1744,7 @@ mod resume_loop_tests {
|
||||
let mut successful = 0;
|
||||
let mut failed = 0;
|
||||
let mut skipped = 0;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
|
||||
let error = healer
|
||||
.heal_bucket_with_resume(
|
||||
@@ -1572,6 +1758,7 @@ mod resume_loop_tests {
|
||||
&mut skipped,
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
.expect_err("a remounted target must not begin a new page scan");
|
||||
@@ -1641,6 +1828,109 @@ mod resume_loop_tests {
|
||||
assert_eq!(skipped, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_progress_accumulates_healed_object_bytes() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"first",
|
||||
Some("v1"),
|
||||
HealResultItem {
|
||||
object_size: 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"second",
|
||||
Some("v2"),
|
||||
HealResultItem {
|
||||
object_size: 2048,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 2);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 2);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.bytes_processed, 3072);
|
||||
assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_written_after_heal_started() {
|
||||
let env = make_env().await;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![
|
||||
item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS),
|
||||
item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1),
|
||||
],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() {
|
||||
let env = make_env().await;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_lifecycle_expired("expired", Some("v1"));
|
||||
|
||||
let (processed, successful, failed, skipped, result) = run(&env).await;
|
||||
|
||||
result.expect("page heal should succeed");
|
||||
assert_eq!(processed, 2);
|
||||
assert_eq!(successful, 1);
|
||||
assert_eq!(failed, 0);
|
||||
assert_eq!(skipped, 0);
|
||||
assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]);
|
||||
assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]);
|
||||
let progress = env.healer.progress.read().await;
|
||||
assert_eq!(progress.skipped_ilm_expired, 1);
|
||||
assert_eq!(progress.objects_scanned, 2);
|
||||
assert_eq!(progress.objects_healed, 1);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_listing_failure_does_not_mark_set_completed() {
|
||||
let env = make_env().await;
|
||||
|
||||
@@ -270,6 +270,8 @@ pub struct HealSourceCounts {
|
||||
pub auto_heal: u64,
|
||||
pub internal: u64,
|
||||
pub read_repair: u64,
|
||||
#[serde(default)]
|
||||
pub mrf: u64,
|
||||
}
|
||||
|
||||
impl HealSourceCounts {
|
||||
@@ -280,6 +282,7 @@ impl HealSourceCounts {
|
||||
HealRequestSource::AutoHeal => self.auto_heal += 1,
|
||||
HealRequestSource::Internal => self.internal += 1,
|
||||
HealRequestSource::ReadRepair => self.read_repair += 1,
|
||||
HealRequestSource::Mrf => self.mrf += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2385,8 +2388,27 @@ impl HealManager {
|
||||
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
|
||||
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
|
||||
snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
|
||||
snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
|
||||
snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
|
||||
snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
|
||||
snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
|
||||
snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
|
||||
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
|
||||
(Some(current), Some(next)) => Some(current.min(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
|
||||
(Some(current), Some(next)) => Some(current.max(next)),
|
||||
(None, next) => next,
|
||||
(current, None) => current,
|
||||
};
|
||||
if progress.current_object.is_some() {
|
||||
snapshot.current_object = progress.current_object;
|
||||
}
|
||||
}
|
||||
snapshot.refresh_progress_percentage();
|
||||
snapshot.refresh_estimated_completion_time();
|
||||
Some(snapshot)
|
||||
}
|
||||
|
||||
@@ -3208,6 +3230,7 @@ impl HealManager {
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
@@ -3223,6 +3246,7 @@ impl HealManager {
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
|
||||
}
|
||||
HealTaskStatus::Retrying { .. } => {}
|
||||
_ => {
|
||||
@@ -3749,6 +3773,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
@@ -5396,6 +5421,8 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = first.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(20));
|
||||
progress.set_total_baseline(12, 8192);
|
||||
progress.update_progress(7, 3, 1, 4096);
|
||||
}
|
||||
|
||||
@@ -5405,6 +5432,8 @@ mod tests {
|
||||
));
|
||||
{
|
||||
let mut progress = second.progress.write().await;
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(8, 4096);
|
||||
progress.update_progress(11, 5, 2, 2048);
|
||||
}
|
||||
|
||||
@@ -5419,7 +5448,11 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 18);
|
||||
assert_eq!(progress.objects_healed, 8);
|
||||
assert_eq!(progress.objects_failed, 3);
|
||||
assert_eq!(progress.objects_total_count, 20);
|
||||
assert_eq!(progress.objects_total_size, 12288);
|
||||
assert_eq!(progress.bytes_processed, 6144);
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod event;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod progress;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Mission Repair Feed (MRF) queue, journal, and consumer.
|
||||
//!
|
||||
//! Intents arriving on the global channel (see `rustfs_common::mrf_channel`)
|
||||
//! are buffered in a bounded in-memory queue, translated into prioritized
|
||||
//! heal requests, and — while they are not yet accepted by the heal manager —
|
||||
//! mirrored into a durable journal so a crash or restart can replay them.
|
||||
//! This is the RustFS counterpart of MinIO's `.heal/mrf/list.bin` replay,
|
||||
//! layered on top of (not replacing) read-repair and scanner heal.
|
||||
//!
|
||||
//! Durability model: the journal is a snapshot of the *unaccepted* pending
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed
|
||||
//! duplicates are merged by the manager's dedup key, and read-repair remains
|
||||
//! the safety net.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::HealManager;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
|
||||
/// Journal location inside the metadata bucket, following the resume-state
|
||||
/// layout.
|
||||
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
|
||||
|
||||
/// Record format tag.
|
||||
const MRF_JOURNAL_FORMAT: u8 = 1;
|
||||
/// Record layout version.
|
||||
const MRF_JOURNAL_VERSION: u8 = 1;
|
||||
|
||||
/// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
|
||||
/// has_version flag.
|
||||
const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct MrfConsumerConfig {
|
||||
/// In-memory queue capacity in intents.
|
||||
pub queue_capacity: usize,
|
||||
/// Journal byte budget; a pending snapshot above this bound is rejected
|
||||
/// oldest-first so the journal can never grow unbounded.
|
||||
pub journal_max_bytes: usize,
|
||||
/// How many journal intents to re-arm per replay round.
|
||||
pub replay_batch: usize,
|
||||
/// Group-commit cadence for the journal snapshot.
|
||||
pub flush_interval: Duration,
|
||||
/// New intents between flushes that force an early snapshot.
|
||||
pub flush_threshold: usize,
|
||||
/// Backoff after the heal manager reports a full admission.
|
||||
pub admission_backoff: Duration,
|
||||
}
|
||||
|
||||
impl Default for MrfConsumerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
queue_capacity: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_QUEUE_SIZE,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_QUEUE_SIZE,
|
||||
),
|
||||
journal_max_bytes: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_JOURNAL_MAX_BYTES,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES,
|
||||
),
|
||||
replay_batch: rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_HEAL_MRF_REPLAY_BATCH,
|
||||
rustfs_config::DEFAULT_HEAL_MRF_REPLAY_BATCH,
|
||||
),
|
||||
flush_interval: Duration::from_millis(500),
|
||||
flush_threshold: 1000,
|
||||
admission_backoff: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded pending set with count and byte ceilings. Overflow drops the
|
||||
/// incoming intent (never a resident one) and counts the loss.
|
||||
pub(crate) struct MrfQueue {
|
||||
pending: VecDeque<MrfIntent>,
|
||||
bytes: usize,
|
||||
capacity: usize,
|
||||
byte_budget: usize,
|
||||
}
|
||||
|
||||
impl MrfQueue {
|
||||
pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
|
||||
Self {
|
||||
pending: VecDeque::new(),
|
||||
bytes: 0,
|
||||
capacity,
|
||||
byte_budget,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `false` (after counting) when either ceiling would be crossed.
|
||||
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
|
||||
let cost = intent.estimated_bytes();
|
||||
if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
return false;
|
||||
}
|
||||
self.bytes += cost;
|
||||
self.pending.push_back(intent);
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
|
||||
let intent = self.pending.pop_front()?;
|
||||
self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
|
||||
Some(intent)
|
||||
}
|
||||
|
||||
pub(crate) fn push_back(&mut self, intent: MrfIntent) {
|
||||
self.bytes += intent.estimated_bytes();
|
||||
self.pending.push_back(intent);
|
||||
}
|
||||
|
||||
pub(crate) fn depth(&self) -> usize {
|
||||
self.pending.len()
|
||||
}
|
||||
|
||||
pub(crate) fn bytes(&self) -> usize {
|
||||
self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn intents(&self) -> impl Iterator<Item = &MrfIntent> {
|
||||
self.pending.iter()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Journal record codec
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Append one encoded record to `out`.
|
||||
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
|
||||
let start = out.len();
|
||||
out.push(MRF_JOURNAL_FORMAT);
|
||||
out.push(MRF_JOURNAL_VERSION);
|
||||
out.push(match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite => 3,
|
||||
});
|
||||
out.push(intent.attempts);
|
||||
out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
|
||||
match intent.version_id {
|
||||
Some(bytes) => {
|
||||
out.push(1);
|
||||
out.extend_from_slice(&bytes);
|
||||
}
|
||||
None => out.push(0),
|
||||
}
|
||||
out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(intent.bucket.as_bytes());
|
||||
out.extend_from_slice(intent.object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&out[start..]);
|
||||
out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
|
||||
}
|
||||
|
||||
fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
|
||||
return None;
|
||||
}
|
||||
if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
|
||||
return None;
|
||||
}
|
||||
let kind = match data[2] {
|
||||
1 => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
2 => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
|
||||
3 => rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
_ => return None,
|
||||
};
|
||||
let attempts = data[3];
|
||||
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
|
||||
let has_version = data[12] != 0;
|
||||
let mut cursor = MRF_RECORD_FIXED_HEAD;
|
||||
let version_id = if has_version {
|
||||
if data.len() < cursor + 16 {
|
||||
return None;
|
||||
}
|
||||
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
|
||||
cursor += 16;
|
||||
Some(bytes)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if data.len() < cursor + 8 {
|
||||
return None;
|
||||
}
|
||||
let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
|
||||
let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
|
||||
cursor += 8;
|
||||
let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
|
||||
let record_end = body_end.checked_add(4)?;
|
||||
if data.len() < record_end {
|
||||
return None;
|
||||
}
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&data[..body_end]);
|
||||
if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
|
||||
return None;
|
||||
}
|
||||
let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
|
||||
let object = std::sync::Arc::from(std::str::from_utf8(&data[cursor + bucket_len..body_end]).ok()?);
|
||||
Some((
|
||||
MrfIntent {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
kind,
|
||||
enqueued_at_ms,
|
||||
attempts,
|
||||
},
|
||||
record_end,
|
||||
))
|
||||
}
|
||||
|
||||
/// Decode a whole journal, stopping at the first torn or corrupt record.
|
||||
/// Returns the decoded intents and the number of trailing bytes discarded.
|
||||
pub(crate) fn decode_journal(data: &[u8]) -> (Vec<MrfIntent>, usize) {
|
||||
let mut intents = Vec::new();
|
||||
let mut cursor = 0usize;
|
||||
while cursor < data.len() {
|
||||
match decode_one(&data[cursor..]) {
|
||||
Some((intent, consumed)) => {
|
||||
intents.push(intent);
|
||||
cursor += consumed;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let truncated = data.len() - cursor;
|
||||
(intents, truncated)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Journal disk IO (all local disks, first successful read wins)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn journal_disks() -> Vec<DiskStore> {
|
||||
let map = local_disk_map_read().await;
|
||||
map.values().flatten().cloned().collect()
|
||||
}
|
||||
|
||||
async fn read_journal() -> Option<Vec<u8>> {
|
||||
for disk in journal_disks().await {
|
||||
match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
|
||||
Ok(bytes) => return Some(bytes.to_vec()),
|
||||
Err(_) => continue,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn write_journal(data: &[u8]) {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
for disk in journal_disks().await {
|
||||
if let Err(err) = disk
|
||||
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
|
||||
.await
|
||||
{
|
||||
warn_mrf_journal_write(&err);
|
||||
}
|
||||
}
|
||||
if !data.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
|
||||
}
|
||||
|
||||
async fn delete_journal() {
|
||||
for disk in journal_disks().await {
|
||||
let _ = disk
|
||||
.delete(
|
||||
super::RUSTFS_META_BUCKET,
|
||||
MRF_JOURNAL_PATH,
|
||||
crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_mrf_journal_write(err: &super::DiskError) {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = %err,
|
||||
"MRF journal write failed; unconsumed intents may be lost on restart"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Consumer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Translate an intent into the prioritized heal request the issue specifies:
|
||||
/// decode failures go Urgent ECDecode, metadata corruption goes High
|
||||
/// Metadata, partial writes go Normal object heal.
|
||||
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
let bucket = intent.bucket.to_string();
|
||||
let object = intent.object.to_string();
|
||||
let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
|
||||
let (heal_type, priority) = match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
HealPriority::Urgent,
|
||||
),
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => (HealType::Metadata { bucket, object }, HealPriority::High),
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite => (
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
};
|
||||
let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
|
||||
request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
|
||||
request
|
||||
}
|
||||
|
||||
struct MrfRuntime {
|
||||
queue: MrfQueue,
|
||||
config: MrfConsumerConfig,
|
||||
new_since_flush: usize,
|
||||
/// True while a journal snapshot exists on disk that no longer reflects
|
||||
/// an all-consumed pending set; the next idle tick removes it (MinIO
|
||||
/// deletes its `list.bin` after replay for the same reason).
|
||||
journal_on_disk: bool,
|
||||
/// Earliest instant a full-admission retry may proceed.
|
||||
backoff_until: Option<tokio::time::Instant>,
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn record_accept(&mut self) {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot, which is the journal's compaction.
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
encode_intent(intent, &mut buf);
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
write_journal(&self.snapshot()).await;
|
||||
self.new_since_flush = 0;
|
||||
self.journal_on_disk = true;
|
||||
}
|
||||
|
||||
/// Drain pending intents into the heal manager until it is full, the
|
||||
/// queue empties, or attempts are exhausted.
|
||||
async fn dispatch(&mut self, manager: &HealManager) {
|
||||
if let Some(until) = self.backoff_until {
|
||||
if tokio::time::Instant::now() < until {
|
||||
return;
|
||||
}
|
||||
self.backoff_until = None;
|
||||
}
|
||||
while let Some(mut intent) = self.queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(),
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff);
|
||||
break;
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) => {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
|
||||
}
|
||||
Err(_) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
|
||||
gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and
|
||||
/// spawn the consumer task. Called once from the heal runtime bootstrap right
|
||||
/// after the manager started; a disabled feature or a double call is a no-op.
|
||||
/// Public for integration tests that drive the real consumer loop.
|
||||
pub fn spawn_mrf_consumer(manager: Arc<HealManager>) {
|
||||
let enabled = rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_MRF_ENABLE, rustfs_config::DEFAULT_HEAL_MRF_ENABLE);
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(enabled);
|
||||
if !enabled {
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF intent pipeline disabled by configuration; producers will not deliver"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let receiver = match rustfs_common::mrf_channel::init_mrf_channel() {
|
||||
Ok(receiver) => receiver,
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
error = err,
|
||||
"MRF channel initialization failed; intents will be dropped at producers"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
run_mrf_consumer(manager, receiver).await;
|
||||
});
|
||||
tracing::info!(target: "rustfs::heal::mrf", "MRF intent consumer started");
|
||||
}
|
||||
|
||||
/// Replay the durable journal into a fresh pending queue and submit whatever
|
||||
/// it armed. Returns the number of intact intents replayed. Duplicates are
|
||||
/// merged by the manager's dedup key; the journal file is removed once read
|
||||
/// (torn tails truncate via the per-record CRC). Public for integration tests;
|
||||
/// the live consumer invokes this through [`replay_into`] at startup.
|
||||
pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
let mut backoff_until: Option<tokio::time::Instant> = None;
|
||||
replay_into(manager, &mut queue, &mut backoff_until).await
|
||||
}
|
||||
|
||||
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
|
||||
async fn replay_into(
|
||||
manager: &Arc<HealManager>,
|
||||
queue: &mut MrfQueue,
|
||||
backoff_until: &mut Option<tokio::time::Instant>,
|
||||
) -> usize {
|
||||
let Some(data) = read_journal().await else {
|
||||
return 0;
|
||||
};
|
||||
let (intents, truncated) = decode_journal(&data);
|
||||
if truncated > 0 {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
truncated_bytes = truncated,
|
||||
"MRF journal had a torn tail; truncated records were discarded"
|
||||
);
|
||||
}
|
||||
counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
|
||||
let replayed = intents.len();
|
||||
for intent in intents {
|
||||
queue.try_push(intent);
|
||||
}
|
||||
delete_journal().await;
|
||||
|
||||
// Drain the replayed intents immediately; whatever the manager refuses
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
if backoff_until.is_none() {
|
||||
while let Some(mut intent) = queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts < MRF_MAX_ATTEMPTS {
|
||||
queue.push_back(intent);
|
||||
*backoff_until = Some(tokio::time::Instant::now());
|
||||
}
|
||||
break;
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) | Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
replayed
|
||||
}
|
||||
|
||||
/// Replay the journal, then keep draining the channel into the heal manager
|
||||
/// while persisting the pending snapshot.
|
||||
async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receiver<MrfIntent>) {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
||||
config: config.clone(),
|
||||
new_since_flush: 0,
|
||||
journal_on_disk: false,
|
||||
backoff_until: None,
|
||||
};
|
||||
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
|
||||
let mut flush_tick = tokio::time::interval(runtime.config.flush_interval);
|
||||
flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
let mut batch: Vec<MrfIntent> = Vec::with_capacity(runtime.config.replay_batch);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
// Channel closed: flush once more and stop.
|
||||
runtime.flush().await;
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF channel closed; consumer stopped after final flush"
|
||||
);
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
runtime.queue.try_push(intent);
|
||||
runtime.new_since_flush += 1;
|
||||
}
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
if runtime.new_since_flush >= runtime.config.flush_threshold {
|
||||
runtime.flush().await;
|
||||
}
|
||||
}
|
||||
_ = flush_tick.tick() => {
|
||||
if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
} else if runtime.journal_on_disk {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind};
|
||||
use std::sync::Arc as StdArc;
|
||||
|
||||
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
||||
MrfIntent {
|
||||
bucket: StdArc::from(bucket),
|
||||
object: StdArc::from(object),
|
||||
version_id: Some([7u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
enqueued_at_ms: 1_700_000_000_000,
|
||||
attempts,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
|
||||
|
||||
let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
|
||||
assert!(tiny.try_push(intent("bucket", "object", 0)));
|
||||
assert!(
|
||||
!tiny.try_push(intent("bucket", "object", 0)),
|
||||
"byte budget must drop before the second intent fits"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_roundtrip_preserves_intents() {
|
||||
let intents = vec![
|
||||
intent("bucket-a", "object/a", 0),
|
||||
intent("bucket-b", "object/b", 2),
|
||||
MrfIntent {
|
||||
bucket: StdArc::from("bucket-c"),
|
||||
object: StdArc::from("object/c"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
enqueued_at_ms: 5,
|
||||
attempts: 1,
|
||||
},
|
||||
];
|
||||
let mut buf = Vec::new();
|
||||
for intent in &intents {
|
||||
encode_intent(intent, &mut buf);
|
||||
}
|
||||
let (decoded, truncated) = decode_journal(&buf);
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(decoded.len(), intents.len());
|
||||
for (left, right) in decoded.iter().zip(intents.iter()) {
|
||||
assert_eq!(left.bucket, right.bucket);
|
||||
assert_eq!(left.object, right.object);
|
||||
assert_eq!(left.version_id, right.version_id);
|
||||
assert_eq!(left.kind, right.kind);
|
||||
assert_eq!(left.attempts, right.attempts);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_torn_tail_is_truncated() {
|
||||
let mut buf = Vec::new();
|
||||
encode_intent(&intent("b", "o", 0), &mut buf);
|
||||
let mut torn = buf.clone();
|
||||
torn.extend_from_slice(&buf[..buf.len() / 2]);
|
||||
|
||||
let (decoded, truncated) = decode_journal(&torn);
|
||||
assert_eq!(decoded.len(), 1, "the intact record must survive");
|
||||
assert!(truncated > 0, "the partial tail must be discarded");
|
||||
|
||||
// A corrupted body (CRC mismatch) also truncates from that record on.
|
||||
let mut corrupt = buf.clone();
|
||||
let mid = MRF_RECORD_FIXED_HEAD + 4;
|
||||
corrupt[mid] ^= 0xff;
|
||||
let (decoded, truncated) = decode_journal(&corrupt);
|
||||
assert!(decoded.is_empty());
|
||||
assert_eq!(truncated, corrupt.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_request_mapping_follows_priority_matrix() {
|
||||
let decode = build_heal_request(&intent("b", "o", 0));
|
||||
assert!(matches!(decode.heal_type, HealType::ECDecode { .. }));
|
||||
assert_eq!(decode.priority, HealPriority::Urgent);
|
||||
|
||||
let metadata = build_heal_request(&MrfIntent {
|
||||
bucket: StdArc::from("b"),
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
assert!(matches!(metadata.heal_type, HealType::Metadata { .. }));
|
||||
assert_eq!(metadata.priority, HealPriority::High);
|
||||
|
||||
let partial = build_heal_request(&MrfIntent {
|
||||
bucket: StdArc::from("b"),
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
assert!(matches!(partial.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(partial.priority, HealPriority::Normal);
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::SystemTime;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -24,6 +24,14 @@ pub struct HealProgress {
|
||||
pub objects_healed: u64,
|
||||
/// Objects failed
|
||||
pub objects_failed: u64,
|
||||
/// Versions skipped because they were written after this heal started
|
||||
pub skipped_new_versions: u64,
|
||||
/// Versions skipped because lifecycle already selected them for expiry
|
||||
pub skipped_ilm_expired: u64,
|
||||
/// Baseline object count from the latest complete usage snapshot
|
||||
pub objects_total_count: u64,
|
||||
/// Baseline object bytes from the latest complete usage snapshot
|
||||
pub objects_total_size: u64,
|
||||
/// Bytes processed
|
||||
pub bytes_processed: u64,
|
||||
/// Current object
|
||||
@@ -54,10 +62,56 @@ impl HealProgress {
|
||||
self.bytes_processed = bytes;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
|
||||
// calculate progress percentage
|
||||
let total = scanned + healed + failed;
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
|
||||
self.objects_total_count = objects_total_count;
|
||||
self.objects_total_size = objects_total_size;
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_new_version(&mut self) {
|
||||
self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
pub fn record_skipped_ilm_expired(&mut self) {
|
||||
self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
self.refresh_progress_percentage();
|
||||
self.refresh_estimated_completion_time();
|
||||
}
|
||||
|
||||
fn completed_for_baseline(&self) -> u64 {
|
||||
self.objects_healed
|
||||
.saturating_add(self.objects_failed)
|
||||
.saturating_add(self.skipped_new_versions)
|
||||
.saturating_add(self.skipped_ilm_expired)
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_progress_percentage(&mut self) {
|
||||
if self.objects_total_size > 0 {
|
||||
self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
if self.objects_total_count > 0 {
|
||||
let completed = self.completed_for_baseline();
|
||||
self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
|
||||
return;
|
||||
}
|
||||
|
||||
let total = self
|
||||
.objects_scanned
|
||||
.saturating_add(self.objects_healed)
|
||||
.saturating_add(self.objects_failed);
|
||||
if total > 0 {
|
||||
self.progress_percentage = (healed as f64 / total as f64) * 100.0;
|
||||
self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,9 +120,36 @@ impl HealProgress {
|
||||
self.last_update_time = Some(SystemTime::now());
|
||||
}
|
||||
|
||||
pub fn refresh_estimated_completion_time(&mut self) {
|
||||
let Some(start_time) = self.start_time else {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
};
|
||||
if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
|
||||
let elapsed = match SystemTime::now().duration_since(start_time) {
|
||||
Ok(elapsed) if !elapsed.is_zero() => elapsed,
|
||||
_ => {
|
||||
self.estimated_completion_time = None;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage;
|
||||
self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs));
|
||||
}
|
||||
|
||||
pub fn is_completed(&self) -> bool {
|
||||
self.progress_percentage >= 100.0
|
||||
|| self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned
|
||||
if self.progress_percentage >= 100.0 {
|
||||
return true;
|
||||
}
|
||||
if self.objects_total_count > 0 || self.objects_total_size > 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
|
||||
}
|
||||
|
||||
pub fn get_success_rate(&self) -> f64 {
|
||||
@@ -158,6 +239,10 @@ mod tests {
|
||||
assert_eq!(progress.objects_scanned, 0);
|
||||
assert_eq!(progress.objects_healed, 0);
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
assert_eq!(progress.skipped_new_versions, 0);
|
||||
assert_eq!(progress.skipped_ilm_expired, 0);
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
assert_eq!(progress.bytes_processed, 0);
|
||||
assert_eq!(progress.progress_percentage, 0.0);
|
||||
assert!(progress.start_time.is_some());
|
||||
@@ -181,6 +266,73 @@ mod tests {
|
||||
assert!(progress.last_update_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_estimates_completion_time_from_progress() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
let eta = progress
|
||||
.estimated_completion_time
|
||||
.expect("partial byte progress should estimate completion");
|
||||
assert!(eta > SystemTime::now());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_byte_baseline_for_percentage() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(100, 25, 0, 4096);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_uses_object_baseline_when_bytes_unknown() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
|
||||
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_counts_skipped_versions_for_object_baseline() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.set_total_baseline(10, 0);
|
||||
|
||||
progress.update_progress(100, 3, 2, 0);
|
||||
progress.record_skipped_new_version();
|
||||
|
||||
assert_eq!(progress.skipped_new_versions, 1);
|
||||
assert!((progress.progress_percentage - 60.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_does_not_estimate_completion_without_bytes() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
|
||||
progress.update_progress(100, 25, 0, 0);
|
||||
|
||||
assert!(progress.estimated_completion_time.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() {
|
||||
let mut progress = HealProgress::new();
|
||||
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
|
||||
progress.set_total_baseline(10, 8192);
|
||||
|
||||
progress.update_progress(1, 1, 0, 1024);
|
||||
|
||||
assert!(!progress.is_completed());
|
||||
assert!(progress.estimated_completion_time.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_progress_update_progress_zero_total() {
|
||||
let mut progress = HealProgress::new();
|
||||
@@ -251,6 +403,8 @@ mod tests {
|
||||
assert_eq!(json["objectsScanned"], 10);
|
||||
assert_eq!(json["objectsHealed"], 8);
|
||||
assert_eq!(json["objectsFailed"], 2);
|
||||
assert_eq!(json["skippedNewVersions"], 0);
|
||||
assert_eq!(json["skippedIlmExpired"], 0);
|
||||
assert_eq!(json["bytesProcessed"], 1024);
|
||||
assert_eq!(json["currentObject"], "test-bucket/test-object");
|
||||
assert!(json["progressPercentage"].is_number());
|
||||
|
||||
@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
|
||||
use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
@@ -29,6 +30,37 @@ use super::storage_api::storage::{
|
||||
use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct HealBucketUsageBaseline {
|
||||
pub objects_count: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
pub struct HealLifecycleExpiryContext {
|
||||
inner: HealLifecycleExpiryContextInner,
|
||||
}
|
||||
|
||||
enum HealLifecycleExpiryContextInner {
|
||||
Ecstore(EcstoreHealLifecycleExpiryContext),
|
||||
#[allow(dead_code)]
|
||||
Test,
|
||||
}
|
||||
|
||||
impl HealLifecycleExpiryContext {
|
||||
fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Ecstore(inner),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test() -> Self {
|
||||
Self {
|
||||
inner: HealLifecycleExpiryContextInner::Test,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
@@ -272,6 +304,10 @@ pub struct HealListItem {
|
||||
pub name: String,
|
||||
/// normalized version id (`None` when the version is nil/absent)
|
||||
pub version_id: Option<String>,
|
||||
/// version modification time as Unix nanoseconds
|
||||
pub mod_time_unix_nanos: Option<i128>,
|
||||
/// object snapshot for lifecycle evaluation
|
||||
pub lifecycle_object_info: Option<HealObjectInfo>,
|
||||
/// whether this version is a delete marker (observability only)
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
@@ -329,6 +365,28 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
/// Aggregate usage-cache baselines for the requested buckets.
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Load per-bucket lifecycle expiry context for heal skips.
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Queue lifecycle expiry for a version that heal can skip.
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
_context: &HealLifecycleExpiryContext,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_version_id: Option<&str>,
|
||||
_object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fix bucket metadata
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
|
||||
|
||||
@@ -409,6 +467,7 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)>;
|
||||
|
||||
/// List versions for healing via a per-erasure-set DISK-WALK union enumerator
|
||||
@@ -427,8 +486,10 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token).await
|
||||
self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Get disk for resume functionality.
|
||||
@@ -1021,6 +1082,85 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if buckets.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await {
|
||||
Ok(info) if info.is_complete_bucket_usage_snapshot() => info,
|
||||
Ok(_) | Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let mut baseline = HealBucketUsageBaseline::default();
|
||||
for bucket in buckets {
|
||||
if let Some(usage) = info.buckets_usage.get(bucket) {
|
||||
baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
|
||||
baseline.bytes = baseline.bytes.saturating_add(usage.size);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(baseline))
|
||||
}
|
||||
|
||||
async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await {
|
||||
Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))),
|
||||
Ok(None) => Ok(None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "load_heal_lifecycle_expiry_context",
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry context load failed"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_heal_lifecycle_expiry(
|
||||
&self,
|
||||
context: &HealLifecycleExpiryContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
object_info: Option<&HealObjectInfo>,
|
||||
) -> Result<bool> {
|
||||
let context = match &context.inner {
|
||||
HealLifecycleExpiryContextInner::Ecstore(context) => context,
|
||||
HealLifecycleExpiryContextInner::Test => return Ok(false),
|
||||
};
|
||||
match self
|
||||
.ecstore
|
||||
.enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info)
|
||||
.await
|
||||
{
|
||||
Ok(queued) => Ok(queued),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "enqueue_heal_lifecycle_expiry",
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
result = "failed",
|
||||
error = %err,
|
||||
"Heal storage lifecycle expiry check failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1436,7 +1576,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
loop {
|
||||
let (page_objects, next_token, is_truncated) = self
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false)
|
||||
.await?;
|
||||
|
||||
all_objects.extend(page_objects);
|
||||
@@ -1471,6 +1611,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1522,10 +1663,19 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let page_objects: Vec<HealListItem> = list_info
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|obj| HealListItem {
|
||||
name: obj.name,
|
||||
version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()),
|
||||
is_delete_marker: obj.delete_marker,
|
||||
.map(|mut obj| {
|
||||
obj.version_id = obj.version_id.filter(|u| !u.is_nil());
|
||||
let version_id = obj.version_id.map(|u| u.to_string());
|
||||
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
|
||||
let is_delete_marker = obj.delete_marker;
|
||||
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info,
|
||||
is_delete_marker,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let page_count = page_objects.len();
|
||||
@@ -1562,6 +1712,7 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
// Per-page bounds for the disk-walk union enumerator. Objects are atomic
|
||||
// (never split across pages), so version_budget only bounds how many
|
||||
@@ -1590,7 +1741,16 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
|
||||
let (versions, next_forward, is_truncated) = self
|
||||
.ecstore
|
||||
.heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET)
|
||||
.heal_walk_versions_page(
|
||||
pool_idx,
|
||||
set_idx,
|
||||
bucket,
|
||||
prefix,
|
||||
forward_to.as_deref(),
|
||||
BATCH_OBJECTS,
|
||||
VERSION_BUDGET,
|
||||
include_lifecycle_object_info,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -1614,6 +1774,8 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map(|v| HealListItem {
|
||||
name: v.name,
|
||||
version_id: v.version_id,
|
||||
mod_time_unix_nanos: v.mod_time_unix_nanos,
|
||||
lifecycle_object_info: v.lifecycle_object_info,
|
||||
is_delete_marker: v.is_delete_marker,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME,
|
||||
load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
@@ -25,7 +28,9 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
@@ -34,8 +39,8 @@ pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
|
||||
ecstore_local_disk_map_read,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,11 +19,12 @@ use crate::heal::{
|
||||
resume::{
|
||||
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
|
||||
},
|
||||
storage::{HealStorageAPI, next_heal_listing_token},
|
||||
storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -178,6 +179,17 @@ pub enum HealPriority {
|
||||
Urgent = 3,
|
||||
}
|
||||
|
||||
impl HealPriority {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Low => "low",
|
||||
Self::Normal => "normal",
|
||||
Self::High => "high",
|
||||
Self::Urgent => "urgent",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealOptions {
|
||||
@@ -498,6 +510,61 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) {
|
||||
trace_emit(|| {
|
||||
let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
||||
.with_duration(duration)
|
||||
.with_attr("task_id", self.id.as_str())
|
||||
.with_attr("heal_type", self.heal_type.log_kind())
|
||||
.with_attr("state", state)
|
||||
.with_attr("source", self.source.as_str())
|
||||
.with_attr("priority", self.priority.as_str())
|
||||
.with_attr("retry_attempts", u64::from(self.retry_attempts))
|
||||
.with_attr("dry_run", self.options.dry_run);
|
||||
|
||||
event = match &self.heal_type {
|
||||
HealType::Cluster => event,
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()),
|
||||
HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()),
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX);
|
||||
event
|
||||
.with_attr("set_disk_id", set_disk_id.as_str())
|
||||
.with_attr("bucket_count", bucket_count)
|
||||
}
|
||||
HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()),
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
let event = event.with_bucket(bucket.as_str()).with_object(object.as_str());
|
||||
match version_id {
|
||||
Some(version_id) => event.with_attr("version_id", version_id.as_str()),
|
||||
None => event,
|
||||
}
|
||||
}
|
||||
HealType::MRF { meta_path } => event.with_object(meta_path.as_str()),
|
||||
};
|
||||
|
||||
match error {
|
||||
Some(error) => event.with_attr("error", error.to_string()),
|
||||
None => event,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn remaining_timeout(&self) -> Result<Option<Duration>> {
|
||||
if let Some(total) = self.options.timeout {
|
||||
let start_instant = { *self.task_start_instant.read().await };
|
||||
@@ -717,6 +784,7 @@ impl HealTask {
|
||||
queue_delay = ?queue_delay,
|
||||
"Heal task started"
|
||||
});
|
||||
self.emit_trace_task_state("started", Duration::ZERO, None);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
@@ -805,6 +873,14 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
let terminal_state = match &result {
|
||||
Ok(_) => "completed",
|
||||
Err(Error::TaskCancelled) => "cancelled",
|
||||
Err(Error::TaskTimeout) => "timed_out",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err());
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1535,7 +1611,7 @@ impl HealTask {
|
||||
let (objects, next_token, is_truncated) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()),
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1697,6 +1773,23 @@ impl HealTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
|
||||
let baseline = match self
|
||||
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
|
||||
.await
|
||||
{
|
||||
Ok(Some(baseline)) => baseline,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total_baseline(objects_count, bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -2298,6 +2391,8 @@ impl HealTask {
|
||||
None
|
||||
};
|
||||
|
||||
self.apply_erasure_set_usage_baseline(&buckets).await?;
|
||||
|
||||
let healing_marker = format!("{set_disk_id}:{}", self.id);
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
let state = resume_manager.get_state().await;
|
||||
@@ -2602,7 +2697,8 @@ impl HealTask {
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
let bytes_processed = progress.bytes_processed;
|
||||
progress.update_progress(4, 4, 0, bytes_processed);
|
||||
}
|
||||
|
||||
match result {
|
||||
@@ -2658,6 +2754,7 @@ mod tests {
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk};
|
||||
use super::*;
|
||||
use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::Mutex;
|
||||
@@ -3203,6 +3300,8 @@ mod tests {
|
||||
block_heal_object: Mutex<bool>,
|
||||
resume_disk: Mutex<Option<DiskStore>>,
|
||||
replacement_resume_disk: Mutex<Option<DiskStore>>,
|
||||
usage_baseline: Mutex<Option<HealBucketUsageBaseline>>,
|
||||
usage_baseline_error: Mutex<bool>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3265,11 +3364,69 @@ mod tests {
|
||||
assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_emits_heal_trace_task_state() {
|
||||
let mut trace = subscribe_trace_events();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let started = recv_trace_task_state(&mut trace, &task.id, "started").await;
|
||||
assert_eq!(started.kind, TraceKind::Heal);
|
||||
assert_eq!(started.func, TraceFunc::HealTask);
|
||||
assert_eq!(started.bucket.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(started.object.as_deref(), Some("object-a"));
|
||||
assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object"));
|
||||
assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal"));
|
||||
assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a"));
|
||||
|
||||
let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await;
|
||||
assert_eq!(completed.kind, TraceKind::Heal);
|
||||
assert_eq!(completed.func, TraceFunc::HealTask);
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), trace.recv())
|
||||
.await
|
||||
.expect("trace event should arrive")
|
||||
.expect("trace bus should stay open");
|
||||
if trace_attr_string(&event, "task_id").as_deref() == Some(task_id)
|
||||
&& trace_attr_string(&event, "state").as_deref() == Some(state)
|
||||
{
|
||||
return (*event).clone();
|
||||
}
|
||||
}
|
||||
|
||||
panic!("expected trace state {state} for task {task_id}");
|
||||
}
|
||||
|
||||
fn trace_attr_string(event: &TraceEvent, key: &str) -> Option<String> {
|
||||
event.attrs.iter().find_map(|attr| {
|
||||
if attr.key != key {
|
||||
return None;
|
||||
}
|
||||
Some(match &attr.value {
|
||||
TraceVal::Bool(value) => value.to_string(),
|
||||
TraceVal::U64(value) => value.to_string(),
|
||||
TraceVal::I64(value) => value.to_string(),
|
||||
TraceVal::Str(value) => value.to_string(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a latest, non-delete-marker heal list item with no version id.
|
||||
fn heal_item(name: &str) -> HealListItem {
|
||||
HealListItem {
|
||||
name: name.to_string(),
|
||||
version_id: None,
|
||||
mod_time_unix_nanos: None,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker: false,
|
||||
}
|
||||
}
|
||||
@@ -3357,6 +3514,13 @@ mod tests {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if *self.usage_baseline_error.lock().unwrap() {
|
||||
return Err(Error::Other("usage baseline unavailable".to_string()));
|
||||
}
|
||||
Ok(*self.usage_baseline.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -3540,6 +3704,7 @@ mod tests {
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
|
||||
if *self.truncate_without_token.lock().unwrap() {
|
||||
@@ -4654,6 +4819,73 @@ mod tests {
|
||||
assert!(storage.object_heal_opts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_applies_usage_baseline_to_progress() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
|
||||
objects_count: 10,
|
||||
bytes: 8,
|
||||
})),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("erasure set heal should complete");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 10);
|
||||
assert_eq!(progress.objects_total_size, 8);
|
||||
assert_eq!(progress.bytes_processed, 2);
|
||||
assert!((progress.progress_percentage - 25.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_ignores_usage_baseline_errors() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
usage_baseline_error: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
|
||||
.await
|
||||
.expect("usage baseline failures should not fail erasure set heal");
|
||||
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_total_count, 0);
|
||||
assert_eq!(progress.objects_total_size, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
|
||||
@@ -158,6 +158,10 @@ pub async fn init_heal_manager_with_workload_provider(
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Start the MRF intent consumer (error-path repair intents + durable
|
||||
// journal replay) now that the manager can accept submissions.
|
||||
heal::mrf_queue::spawn_mrf_consumer(heal_manager.clone());
|
||||
|
||||
#[cfg(test)]
|
||||
test_hook_after_manager_start().await;
|
||||
|
||||
@@ -445,6 +449,7 @@ mod tests {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool), Error> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc<ECStoreHealStorage>, bucket:
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("list_objects_for_heal_page failed");
|
||||
items.extend(page);
|
||||
|
||||
@@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc<ECStoreHealStorage>, bucket: &str) -> V
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref())
|
||||
.list_objects_for_heal_page(bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("b5 list page failed");
|
||||
items.extend(page);
|
||||
@@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc<ECStoreHealStorage>, bucket: &st
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let (page, next, truncated) = heal_storage
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref())
|
||||
.list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false)
|
||||
.await
|
||||
.expect("disk-walk list page failed");
|
||||
items.extend(page);
|
||||
@@ -418,7 +418,7 @@ mod serial_tests {
|
||||
let mut pages = 0usize;
|
||||
loop {
|
||||
let (versions, next_forward, truncated) = ecstore
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000)
|
||||
.heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false)
|
||||
.await
|
||||
.expect("heal_walk_versions_page failed");
|
||||
pages += 1;
|
||||
|
||||
@@ -242,6 +242,7 @@ fn test_heal_task_status_atomic_update() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
@@ -385,6 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
_continuation_token: Option<&str>,
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> rustfs_heal::Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! HS-01 (rustfs/backlog#1865): MRF intent pipeline integration tests.
|
||||
//!
|
||||
//! Drives the real consumer loop (`spawn_mrf_consumer`) against a real
|
||||
//! 4-disk `ECStore` heal storage and a `HealManager` that has not started its
|
||||
//! scheduler, so submitted intents stay observable in the admission queue.
|
||||
//! Under `cargo nextest` each test runs in its own process, which keeps the
|
||||
//! process-global MRF channel singleton safe.
|
||||
|
||||
use rustfs_common::mrf_channel::{self, MrfKind};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
mrf_queue,
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{path::Path, sync::Arc, time::Duration};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
|
||||
const META_BUCKET: &str = ".rustfs.sys";
|
||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_heal_mrf_test")
|
||||
.build()
|
||||
.await;
|
||||
let heal_storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
(env.disk_paths, heal_storage)
|
||||
}
|
||||
|
||||
fn make_manager(storage: Arc<dyn HealStorageAPI>) -> Arc<HealManager> {
|
||||
Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
// Keep the scheduler from draining the queue before assertions.
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
/// Encode one journal record independently of the implementation, so a format
|
||||
/// drift between writer and this fixture fails loudly here.
|
||||
fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]>, attempts: u8) -> Vec<u8> {
|
||||
let mut body = vec![1u8, 1, kind, attempts];
|
||||
body.extend_from_slice(&1_700_000_000_000u64.to_le_bytes());
|
||||
match version {
|
||||
Some(bytes) => {
|
||||
body.push(1);
|
||||
body.extend_from_slice(&bytes);
|
||||
}
|
||||
None => body.push(0),
|
||||
}
|
||||
body.extend_from_slice(&(bucket.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(&(object.len() as u32).to_le_bytes());
|
||||
body.extend_from_slice(bucket.as_bytes());
|
||||
body.extend_from_slice(object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&body);
|
||||
body.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
|
||||
body
|
||||
}
|
||||
|
||||
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
for path in disk_paths {
|
||||
let journal = path.join(META_BUCKET).join(JOURNAL_REL);
|
||||
std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
|
||||
std::fs::write(&journal, data).expect("write journal fixture");
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: std::future::Future<Output = bool>,
|
||||
{
|
||||
let start = std::time::Instant::now();
|
||||
while start.elapsed() < deadline {
|
||||
if probe().await {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// A decode-failure intent delivered on the global channel must surface in the
|
||||
/// heal manager as an Urgent request attributed to the MRF source.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
||||
let (_disk_paths, storage) = heal_env().await;
|
||||
let manager = make_manager(storage);
|
||||
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
|
||||
assert!(
|
||||
mrf_channel::try_send_mrf_intent(MrfKind::DecodeFailure, "mrf-bucket", "mrf-object", None),
|
||||
"intent should be accepted while the consumer holds the channel"
|
||||
);
|
||||
|
||||
let appeared = wait_until(Duration::from_secs(10), || async {
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
snapshot.queued_by_source.mrf >= 1 && snapshot.queued_by_priority.urgent >= 1
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
appeared,
|
||||
"MRF intent must reach the manager queue as an Urgent request (snapshot: {:?})",
|
||||
manager.operations_snapshot().await
|
||||
);
|
||||
}
|
||||
|
||||
/// A journal left behind by a previous process must be replayed into the
|
||||
/// manager queue and then removed, and a torn tail must not block replay of
|
||||
/// the intact records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
|
||||
// The journal reader resolves disks through the process-local disk map;
|
||||
// register the environment's disks the same way server startup does.
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "mrf-test".to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
|
||||
let mut journal = journal_record(1, "replay-bucket", "replay-object", Some([9u8; 16]), 0);
|
||||
journal.extend(journal_record(3, "replay-bucket", "partial-object", None, 1));
|
||||
// Torn tail: a third record truncated mid-way must not block the two
|
||||
// intact records above.
|
||||
journal.extend_from_slice(&journal_record(2, "replay-bucket", "metadata-object", None, 0)[..8]);
|
||||
write_journal_to_disks(&disk_paths, &journal);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
// Replay directly (not via the process-global channel consumer, which the
|
||||
// sibling test already claimed in this process under plain `cargo test`).
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 2, "the two intact records must be replayed");
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 2, "replayed intents must be attributed to the MRF source");
|
||||
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"the journal file must be removed after a successful replay"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
|
||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||
}
|
||||
@@ -26,7 +26,6 @@ pub enum StorageMedia {
|
||||
}
|
||||
|
||||
impl StorageMedia {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Nvme => "nvme",
|
||||
@@ -60,7 +59,6 @@ pub enum AccessPattern {
|
||||
}
|
||||
|
||||
impl AccessPattern {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Sequential => "sequential",
|
||||
@@ -71,25 +69,21 @@ impl AccessPattern {
|
||||
}
|
||||
|
||||
/// Check if this is a sequential access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_sequential(&self) -> bool {
|
||||
matches!(self, Self::Sequential)
|
||||
}
|
||||
|
||||
/// Check if this is a random access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_random(&self) -> bool {
|
||||
matches!(self, Self::Random)
|
||||
}
|
||||
|
||||
/// Check if this is a mixed access pattern.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_mixed(&self) -> bool {
|
||||
matches!(self, Self::Mixed)
|
||||
}
|
||||
|
||||
/// Check if this pattern is unknown.
|
||||
#[allow(dead_code)]
|
||||
pub fn is_unknown(&self) -> bool {
|
||||
matches!(self, Self::Unknown)
|
||||
}
|
||||
|
||||
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
Kubernetes {
|
||||
role: String,
|
||||
#[serde(default)]
|
||||
mount: Option<String>,
|
||||
#[serde(default)]
|
||||
jwt_path: Option<std::path::PathBuf>,
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
TokenFile {
|
||||
path: std::path::PathBuf,
|
||||
#[serde(default)]
|
||||
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => Self::Kubernetes {
|
||||
role,
|
||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||
refresh_safety_window_secs,
|
||||
},
|
||||
StrictVaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
auth_method_type: match &vault_config.auth_method {
|
||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||
},
|
||||
has_stored_credentials: true,
|
||||
@@ -901,6 +923,42 @@ mod tests {
|
||||
assert!(request.to_kms_config().validate().is_ok());
|
||||
}
|
||||
|
||||
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||
/// manifest carries no credential and no cluster-specific paths.
|
||||
#[test]
|
||||
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||
let raw = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"mount_path": "rustfs",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||
});
|
||||
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||
let config = request.to_kms_config();
|
||||
config.validate().expect("kubernetes auth must validate");
|
||||
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
|
||||
let unknown_field = serde_json::json!({
|
||||
"backend_type": "vault-transit",
|
||||
"address": "https://vault.example.com:8200",
|
||||
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||
});
|
||||
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||
|
||||
@@ -550,6 +550,7 @@ impl VaultKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||
///
|
||||
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||
///
|
||||
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||
/// the projected token and mounts it world-readable by default, so rejecting
|
||||
/// group/other bits would refuse every standard pod rather than catch a
|
||||
/// deployment error.
|
||||
pub(crate) struct KubernetesLogin {
|
||||
/// Unauthenticated client used only for the login exchange.
|
||||
login_client: VaultClient,
|
||||
mount: String,
|
||||
role: String,
|
||||
jwt_path: PathBuf,
|
||||
}
|
||||
|
||||
impl KubernetesLogin {
|
||||
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||
Ok(Self {
|
||||
login_client: settings.build_login_client()?,
|
||||
mount,
|
||||
role,
|
||||
jwt_path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the ServiceAccount token for one login attempt.
|
||||
///
|
||||
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||
/// has not projected yet heals the source without a restart.
|
||||
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||
.await
|
||||
.map_err(|error| AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
})?;
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
raw.zeroize();
|
||||
return Err(AttemptError {
|
||||
class: ErrorClass::Fatal,
|
||||
error: KmsError::configuration_error(format!(
|
||||
"Kubernetes ServiceAccount token {} is empty",
|
||||
self.jwt_path.display()
|
||||
)),
|
||||
});
|
||||
}
|
||||
let jwt = SecretString::new(trimmed.to_string());
|
||||
raw.zeroize();
|
||||
Ok(jwt)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TokenSource for KubernetesLogin {
|
||||
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||
let jwt = self.resolve_jwt().await?;
|
||||
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||
.await
|
||||
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
|
||||
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||
let auth = vaultrs::token::renew_self(client, None)
|
||||
.await
|
||||
.map_err(|error| attempt_error("token renewal", error))?;
|
||||
Ok(TokenLease::from_auth(auth))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for KubernetesLogin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// The login client embeds Vault client settings and must stay out of
|
||||
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||
f.debug_struct("KubernetesLogin")
|
||||
.field("mount", &self.mount)
|
||||
.field("role", &self.role)
|
||||
.field("jwt_path", &self.jwt_path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||
/// token file (for example a Vault Agent auto-auth sink).
|
||||
///
|
||||
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
|
||||
secret_id.clone(),
|
||||
secret_id_file.clone(),
|
||||
)?)),
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
|
||||
pub(crate) namespace: Option<String>,
|
||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||
pub(crate) attempt_timeout: Duration,
|
||||
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||
pub(crate) skip_tls_verify: bool,
|
||||
}
|
||||
|
||||
impl VaultConnectionSettings {
|
||||
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
|
||||
// operation-level retry policy.
|
||||
settings_builder.timeout(Some(self.attempt_timeout));
|
||||
settings_builder.token(token);
|
||||
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||
// disable certificate verification behind the KMS configuration and its
|
||||
// insecure-defaults gate.
|
||||
settings_builder.verify(!self.skip_tls_verify);
|
||||
|
||||
if let Some(namespace) = &self.namespace {
|
||||
settings_builder.namespace(Some(namespace.clone()));
|
||||
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::Kubernetes {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
}
|
||||
| VaultAuthMethod::TokenFile {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
..
|
||||
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
|
||||
|
||||
impl VaultClientHandle {
|
||||
/// Absolute expiry of this generation's token.
|
||||
///
|
||||
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||
/// collapses to `None` — the same answer already given for the zero-lease
|
||||
/// tokens Vault issues, which keeps the token in use and still fully
|
||||
/// validated by Vault on every call.
|
||||
fn expires_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||
}
|
||||
|
||||
/// When the renewal task should refresh this generation: half the TTL,
|
||||
/// leaving the second half as budget for retries before the fail-closed
|
||||
/// window is reached.
|
||||
///
|
||||
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||
/// leaving a token that never expires with nothing to renew.
|
||||
fn renew_at(&self) -> Option<Instant> {
|
||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
||||
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
|
||||
let handle = self.current.load_full();
|
||||
if let Some(expires_at) = handle.expires_at() {
|
||||
let now = Instant::now();
|
||||
if now + self.policy.safety_window >= expires_at {
|
||||
if self.inside_safety_window(now, expires_at) {
|
||||
return Err(KmsError::credentials_unavailable(format!(
|
||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||
handle.generation, self.policy.safety_window
|
||||
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||
///
|
||||
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||
/// guaranteed to have passed this version's validation: a window too large
|
||||
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||
/// means every token is always inside it, so saturating to "refuse" is both
|
||||
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||
now.checked_add(self.policy.safety_window)
|
||||
.is_none_or(|deadline| deadline >= expires_at)
|
||||
}
|
||||
|
||||
/// Publish the credential gauges for the generation currently installed.
|
||||
///
|
||||
/// The fail-closed gauge re-evaluates the very gate
|
||||
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
|
||||
let fail_closed = match handle.expires_at() {
|
||||
Some(expires_at) => {
|
||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||
now + self.policy.safety_window >= expires_at
|
||||
self.inside_safety_window(now, expires_at)
|
||||
}
|
||||
// A generation without an expiry has no remaining TTL to report
|
||||
// and can never lapse, so it can never fail closed either.
|
||||
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::REDACTED_SECRET;
|
||||
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
|
||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||
@@ -871,6 +999,7 @@ mod tests {
|
||||
address: "http://127.0.0.1:8200".to_string(),
|
||||
namespace: Some("team-namespace".to_string()),
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1057,6 +1186,143 @@ mod tests {
|
||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||
let settings = test_settings();
|
||||
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||
.expect("kubernetes auth must map to a login source");
|
||||
|
||||
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||
}
|
||||
|
||||
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||
/// path from persisted configuration, so the fail-closed comparison must
|
||||
/// survive a window too large to add to the current instant. Before the
|
||||
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||
/// instant" on the first request after a lease-bearing login.
|
||||
#[tokio::test]
|
||||
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = provider
|
||||
.current()
|
||||
.expect_err("a window wider than any lease must refuse the token");
|
||||
assert!(
|
||||
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||
"expected CredentialsUnavailable, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||
/// the other side of the same comparison, so an absurd one must not panic
|
||||
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||
/// the zero-lease case already behaves.
|
||||
#[tokio::test]
|
||||
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(u64::MAX),
|
||||
true,
|
||||
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
|
||||
provider
|
||||
.current()
|
||||
.expect("a token whose expiry cannot be represented must stay usable");
|
||||
}
|
||||
|
||||
/// The configured flag has to reach the HTTP client, not just the config
|
||||
/// struct: every generation (authenticated and login) builds its own client,
|
||||
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||
/// each one carries the setting.
|
||||
#[test]
|
||||
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||
for skip_tls_verify in [false, true] {
|
||||
let settings = VaultConnectionSettings {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
namespace: None,
|
||||
attempt_timeout: Duration::from_secs(30),
|
||||
skip_tls_verify,
|
||||
};
|
||||
|
||||
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||
|
||||
let login = settings.build_login_client().expect("login client must build");
|
||||
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||
}
|
||||
}
|
||||
|
||||
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||
/// builder leaves it unset, which would disable certificate verification
|
||||
/// without passing the KMS insecure-defaults gate.
|
||||
#[test]
|
||||
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||
assert!(
|
||||
client.settings.verify,
|
||||
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||
/// does not corrupt the assertion sent to Vault.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("token");
|
||||
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||
|
||||
let login = KubernetesLogin::new(
|
||||
&test_settings(),
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
"rustfs".to_string(),
|
||||
path.clone(),
|
||||
)
|
||||
.expect("login source must build");
|
||||
|
||||
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||
|
||||
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||
assert_eq!(
|
||||
login.resolve_jwt().await.expect("second read").expose(),
|
||||
"rotated-jwt",
|
||||
"a rotated projected token must be picked up without a restart"
|
||||
);
|
||||
}
|
||||
|
||||
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||
/// keeps retrying, which is what lets a late projection heal the source.
|
||||
#[tokio::test]
|
||||
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let missing = dir.path().join("absent-token");
|
||||
let empty = dir.path().join("empty-token");
|
||||
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||
|
||||
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||
let login =
|
||||
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||
.expect("login source must build");
|
||||
|
||||
let error = login
|
||||
.acquire()
|
||||
.await
|
||||
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_renewal_task_renews_at_half_ttl() {
|
||||
let (provider, state) = scripted_provider(
|
||||
|
||||
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
|
||||
address: config.address.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
|
||||
address: target.address.clone(),
|
||||
namespace: target.namespace.clone(),
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
// A restore target carries no TLS settings, so certificates are
|
||||
// always verified: recovery is the last path that should accept an
|
||||
// unauthenticated Vault.
|
||||
skip_tls_verify: false,
|
||||
};
|
||||
let source = token_source_for(&target.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
|
||||
+295
-54
@@ -25,6 +25,10 @@ use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
|
||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||
///
|
||||
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
|
||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_mount() -> String {
|
||||
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||
}
|
||||
|
||||
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||
}
|
||||
|
||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||
/// for a lease-bound Vault token that is renewed in the background.
|
||||
Kubernetes {
|
||||
/// Vault role bound to this ServiceAccount.
|
||||
role: String,
|
||||
/// Kubernetes auth engine mount path.
|
||||
#[serde(default = "default_vault_kubernetes_mount")]
|
||||
mount: String,
|
||||
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||
/// a token the kubelet rotates is picked up without a restart.
|
||||
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||
jwt_path: PathBuf,
|
||||
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||
/// per-attempt timeout.
|
||||
#[serde(default)]
|
||||
refresh_safety_window_secs: Option<u64>,
|
||||
},
|
||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||
/// the token is read from `path` and re-read periodically so a token
|
||||
/// rotated by the agent is picked up without a restart.
|
||||
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kubernetes authentication with the default mount and projected token path.
|
||||
pub fn kubernetes(role: String) -> Self {
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount: default_vault_kubernetes_mount(),
|
||||
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||
refresh_safety_window_secs: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Agent-managed token file with the default poll interval.
|
||||
pub fn token_file(path: PathBuf) -> Self {
|
||||
Self::TokenFile {
|
||||
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
|
||||
.field("mount", mount)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
// No redaction: the role and mount name a Vault binding, and the
|
||||
// ServiceAccount token itself is never held on this type.
|
||||
Self::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} => f
|
||||
.debug_struct("Kubernetes")
|
||||
.field("role", role)
|
||||
.field("mount", mount)
|
||||
.field("jwt_path", jwt_path)
|
||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||
.finish(),
|
||||
Self::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1028,50 +1087,12 @@ impl KmsConfig {
|
||||
});
|
||||
}
|
||||
KmsBackend::VaultKv2 => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::VaultTransit => {
|
||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
||||
let auth_method = vault_auth_method_from_env()?;
|
||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
||||
|
||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address,
|
||||
auth_method,
|
||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
||||
metadata_kv_mount: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
),
|
||||
metadata_key_prefix: get_env_str(
|
||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
||||
),
|
||||
tls: vault_tls_config(skip_tls_verify),
|
||||
}));
|
||||
config.backend_config =
|
||||
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||
}
|
||||
KmsBackend::Static => {
|
||||
// Read from file first, then fall back to direct env var
|
||||
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
path.starts_with(std::env::temp_dir())
|
||||
}
|
||||
|
||||
/// Command-line values that take precedence over the matching environment
|
||||
/// variables when assembling a Vault backend configuration.
|
||||
///
|
||||
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||
/// reads, so these are only set when the operator passed an explicit flag.
|
||||
///
|
||||
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||
/// error instead of a leak.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct VaultCliOverrides<'a> {
|
||||
pub address: Option<&'a str>,
|
||||
pub token: Option<&'a str>,
|
||||
pub mount_path: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||
///
|
||||
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||
let mount_path = match overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||
{
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||
);
|
||||
path
|
||||
}
|
||||
None => default_vault_kv2_mount_path(),
|
||||
};
|
||||
|
||||
Ok(VaultConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path,
|
||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Assemble the Vault Transit backend configuration from the environment.
|
||||
///
|
||||
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||
/// points share it.
|
||||
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||
Ok(VaultTransitConfig {
|
||||
address: vault_address_from_env(overrides.address),
|
||||
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||
mount_path: overrides
|
||||
.mount_path
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||
})
|
||||
}
|
||||
|
||||
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||
override_value
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||
}
|
||||
|
||||
/// Resolve the Vault auth method from environment variables.
|
||||
///
|
||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
||||
///
|
||||
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||
/// presents the pod's projected ServiceAccount token.
|
||||
///
|
||||
/// `token_override` carries a token supplied on the command line; it stands in
|
||||
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||
/// so a flag and the variable it mirrors select the same method.
|
||||
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||
let token = token_override
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||
|
||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||
// A token file names one authoritative credential source; combining it
|
||||
// with another one would leave the effective identity ambiguous, so
|
||||
// that is a configuration error rather than a precedence rule.
|
||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
||||
)));
|
||||
for (name, configured) in [
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||
] {
|
||||
if configured {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
}
|
||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||
}
|
||||
|
||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
||||
if let Some(role) = kubernetes_role {
|
||||
// Unlike a leftover static token, a second login method is never a
|
||||
// stale remnant: both were configured deliberately and neither can be
|
||||
// ranked over the other.
|
||||
if role_id.is_some() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||
)));
|
||||
}
|
||||
return Ok(VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||
refresh_safety_window_secs: None,
|
||||
});
|
||||
}
|
||||
|
||||
let Some(role_id) = role_id else {
|
||||
return Ok(VaultAuthMethod::Token {
|
||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
||||
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::Kubernetes {
|
||||
role, mount, jwt_path, ..
|
||||
} => {
|
||||
if role.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||
}
|
||||
if mount.is_empty() {
|
||||
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||
}
|
||||
if jwt_path.as_os_str().is_empty() {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VaultAuthMethod::TokenFile {
|
||||
path,
|
||||
poll_interval_secs,
|
||||
@@ -1976,6 +2117,106 @@ mod tests {
|
||||
.expect("well-formed token file auth must validate");
|
||||
}
|
||||
|
||||
/// A Kubernetes role alone configures the method: the credential is the
|
||||
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||
/// environment and the mount and token path fall back to the cluster
|
||||
/// defaults.
|
||||
#[test]
|
||||
fn test_from_env_selects_kubernetes() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||
let VaultAuthMethod::Kubernetes {
|
||||
role,
|
||||
mount,
|
||||
jwt_path,
|
||||
refresh_safety_window_secs,
|
||||
} = &vault.auth_method
|
||||
else {
|
||||
panic!(
|
||||
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||
vault.auth_method
|
||||
);
|
||||
};
|
||||
assert_eq!(role, "rustfs");
|
||||
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||
assert_eq!(refresh_safety_window_secs, &None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||
(ENV_KMS_VAULT_TOKEN, None),
|
||||
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||
],
|
||||
|| {
|
||||
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||
backend: KmsBackend::VaultTransit,
|
||||
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address: "https://vault.example.com:8200".to_string(),
|
||||
auth_method,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes role must be rejected");
|
||||
assert!(error.to_string().contains("role"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: String::new(),
|
||||
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty kubernetes mount must be rejected");
|
||||
assert!(error.to_string().contains("mount"), "got {error}");
|
||||
|
||||
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||
role: "rustfs".to_string(),
|
||||
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||
jwt_path: PathBuf::new(),
|
||||
refresh_safety_window_secs: None,
|
||||
})
|
||||
.validate()
|
||||
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||
assert!(error.to_string().contains("token path"), "got {error}");
|
||||
|
||||
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||
.validate()
|
||||
.expect("well-formed kubernetes auth must validate");
|
||||
}
|
||||
|
||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||
/// empty one names a path no Vault engine answers. The Transit backend
|
||||
/// already rejects its own empty mounts; this closes the same gap on the
|
||||
|
||||
@@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
rustfs-signer.workspace = true
|
||||
s3s.workspace = true
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
@@ -49,3 +53,4 @@ doctest = false
|
||||
|
||||
[dev-dependencies]
|
||||
rmp-serde.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] }
|
||||
|
||||
@@ -0,0 +1,851 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869).
|
||||
//!
|
||||
//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4
|
||||
//! request signing (the same scheme the server's admin router authenticates),
|
||||
//! so `mc`-style tooling and automation can drive heal start/query/cancel and
|
||||
//! read background-heal / scanner status without hand-rolling HTTP.
|
||||
//!
|
||||
//! Wire structs in this module mirror the server-side shapes
|
||||
//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`,
|
||||
//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where
|
||||
//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply
|
||||
//! nested status payloads that the server composes from runtime types are
|
||||
//! carried through as `serde_json::Value` and flattened maps rather than
|
||||
//! duplicated field-for-field, so the client cannot silently drift on fields
|
||||
//! it never interprets.
|
||||
|
||||
use crate::heal_commands::HealResultItem;
|
||||
use http::Method;
|
||||
use serde::{Deserialize, Serialize, de};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default admin API path prefix on a RustFS endpoint.
|
||||
pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin";
|
||||
/// Default SigV4 region when the server has no explicit region configured.
|
||||
pub const DEFAULT_REGION: &str = "us-east-1";
|
||||
|
||||
/// Scan mode for a heal request, mirroring the server's numeric-or-name wire
|
||||
/// encoding (`0` unknown/default, `1` normal, `2` deep).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealScanMode {
|
||||
/// Server default; behaves as [`HealScanMode::Normal`].
|
||||
#[default]
|
||||
Unknown,
|
||||
/// Metadata-level checks only.
|
||||
Normal,
|
||||
/// Full bitrot verification while healing.
|
||||
Deep,
|
||||
}
|
||||
|
||||
impl HealScanMode {
|
||||
fn wire_number(self) -> u8 {
|
||||
match self {
|
||||
Self::Unknown => 0,
|
||||
Self::Normal => 1,
|
||||
Self::Deep => 2,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_number(value: u8) -> Option<Self> {
|
||||
match value {
|
||||
0 => Some(Self::Unknown),
|
||||
1 => Some(Self::Normal),
|
||||
2 => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_wire_name(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"normal" => Some(Self::Normal),
|
||||
"deep" => Some(Self::Deep),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for HealScanMode {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_u8(self.wire_number())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for HealScanMode {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct HealScanModeVisitor;
|
||||
|
||||
impl de::Visitor<'_> for HealScanModeVisitor {
|
||||
type Value = HealScanMode;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("a heal scan mode number or name")
|
||||
}
|
||||
|
||||
fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
|
||||
u8::try_from(value)
|
||||
.ok()
|
||||
.and_then(HealScanMode::from_wire_number)
|
||||
.ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}")))
|
||||
}
|
||||
|
||||
fn visit_str<E: de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}")))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(HealScanModeVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal options for an admin heal request (mirror of the server body type).
|
||||
/// Fields default on decode: a client should tolerate a server response whose
|
||||
/// settings object omits fields it never set.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct HealOpts {
|
||||
#[serde(default)]
|
||||
pub recursive: bool,
|
||||
#[serde(rename = "dryRun", default)]
|
||||
pub dry_run: bool,
|
||||
#[serde(default)]
|
||||
pub remove: bool,
|
||||
#[serde(default)]
|
||||
pub recreate: bool,
|
||||
#[serde(rename = "scanMode", default)]
|
||||
pub scan_mode: HealScanMode,
|
||||
#[serde(rename = "updateParity", default)]
|
||||
pub update_parity: bool,
|
||||
#[serde(rename = "nolock", default)]
|
||||
pub no_lock: bool,
|
||||
#[serde(rename = "pool", default)]
|
||||
pub pool: Option<usize>,
|
||||
#[serde(rename = "set", default)]
|
||||
pub set: Option<usize>,
|
||||
}
|
||||
|
||||
/// Successful heal start / path-scoped cancel response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealStartSuccess {
|
||||
pub client_token: String,
|
||||
pub client_address: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
}
|
||||
|
||||
/// Heal task status response (query, cancel-with-token, start-then-poll).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealTaskStatus {
|
||||
/// `running` | `finished` | `stopped` | `notFound`.
|
||||
pub summary: String,
|
||||
/// Failure detail for stopped tasks; empty otherwise.
|
||||
#[serde(rename = "detail", default)]
|
||||
pub failure_detail: String,
|
||||
#[serde(default)]
|
||||
pub start_time: String,
|
||||
#[serde(default)]
|
||||
pub settings: HealOpts,
|
||||
#[serde(default)]
|
||||
pub items: Vec<HealResultItem>,
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
/// Live progress snapshot; the exact shape is owned by the heal runtime.
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `POST /v3/background-heal/status` response. Known top-level fields are
|
||||
/// typed; the flattened heal info and operations matrix pass through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealStatus {
|
||||
/// `disabled` | `uninitialized` | `idle` | `active` | `degraded`.
|
||||
pub state: String,
|
||||
#[serde(default)]
|
||||
pub heal_queue_length: u64,
|
||||
#[serde(default)]
|
||||
pub heal_active_tasks: u64,
|
||||
#[serde(default)]
|
||||
pub cluster_status_complete: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
|
||||
/// priority-by-source operations matrix), carried verbatim.
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// `GET /v3/scanner/status` response, typed at the fields operators branch
|
||||
/// on; everything else passes through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerStatus {
|
||||
pub enabled: bool,
|
||||
/// `fresh` | `stale` | `unknown`; absent when the scanner never completed
|
||||
/// a cycle.
|
||||
#[serde(default)]
|
||||
pub freshness: Option<ScannerFreshness>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Freshness block of the scanner status response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScannerFreshness {
|
||||
/// `fresh` | `stale` | `unknown`.
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
impl ScannerStatus {
|
||||
/// Convenience accessor for the freshness state string.
|
||||
pub fn freshness(&self) -> &str {
|
||||
self.freshness
|
||||
.as_ref()
|
||||
.map(|freshness| freshness.state.as_str())
|
||||
.unwrap_or("unknown")
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything that can go wrong in an admin client call.
|
||||
#[derive(Debug)]
|
||||
pub enum AdminClientError {
|
||||
/// The endpoint URL could not be parsed.
|
||||
InvalidEndpoint(String),
|
||||
/// Request build/send failed (DNS, connect, timeout, body read).
|
||||
Transport(reqwest::Error),
|
||||
/// The server answered a non-2xx status.
|
||||
HttpStatus { status: u16, body: String },
|
||||
/// The response body did not decode into the expected shape.
|
||||
Decode { message: String },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for AdminClientError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"),
|
||||
Self::Transport(err) => write!(f, "admin request transport failure: {err}"),
|
||||
Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"),
|
||||
Self::Decode { message } => write!(f, "admin response decode failure: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AdminClientError {}
|
||||
|
||||
impl From<reqwest::Error> for AdminClientError {
|
||||
fn from(err: reqwest::Error) -> Self {
|
||||
Self::Transport(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed client for a RustFS admin API.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdminClient {
|
||||
endpoint: reqwest::Url,
|
||||
access_key: String,
|
||||
secret_key: String,
|
||||
session_token: String,
|
||||
region: String,
|
||||
api_prefix: String,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl AdminClient {
|
||||
/// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root
|
||||
/// or admin credentials. Requests are SigV4-signed with the same scheme
|
||||
/// the server's admin router authenticates.
|
||||
pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result<Self, AdminClientError> {
|
||||
let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if url.host_str().is_none() {
|
||||
return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string()));
|
||||
}
|
||||
let http = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
Ok(Self {
|
||||
endpoint: url,
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
session_token: String::new(),
|
||||
region: DEFAULT_REGION.to_string(),
|
||||
api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(),
|
||||
http,
|
||||
})
|
||||
}
|
||||
|
||||
/// Attach an STS session token (signed as `x-amz-security-token`).
|
||||
pub fn with_session_token(mut self, session_token: impl Into<String>) -> Self {
|
||||
self.session_token = session_token.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the SigV4 region (defaults to `us-east-1`, matching a
|
||||
/// region-less RustFS deployment).
|
||||
pub fn with_region(mut self, region: impl Into<String>) -> Self {
|
||||
self.region = region.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the admin API path prefix (defaults to `/rustfs/admin`).
|
||||
pub fn with_api_prefix(mut self, prefix: impl Into<String>) -> Self {
|
||||
self.api_prefix = prefix.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Start a heal. `bucket` empty and `prefix` empty heals the whole
|
||||
/// deployment (requires `recursive` or a `pool`/`set` pair in `opts`,
|
||||
/// enforced server-side); a bucket alone heals the bucket (the server
|
||||
/// forces `recursive` for bucket heals).
|
||||
pub async fn heal_start(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
force_start: bool,
|
||||
) -> Result<HealStartSuccess, AdminClientError> {
|
||||
let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
let mut query = Vec::new();
|
||||
if force_start {
|
||||
query.push(("forceStart", "true".to_string()));
|
||||
}
|
||||
self.post_json(&heal_path(bucket, prefix), &query, body).await
|
||||
}
|
||||
|
||||
/// Query the status of the heal identified by `client_token` (the token
|
||||
/// returned by [`Self::heal_start`]) at the path it was started on.
|
||||
pub async fn heal_status(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: &str,
|
||||
) -> Result<HealTaskStatus, AdminClientError> {
|
||||
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stop a heal: with a `client_token` only that task is cancelled and its
|
||||
/// final status returned; without one, every heal task at the path is
|
||||
/// cancelled (the server answers with a start-success-shaped receipt).
|
||||
pub async fn heal_stop(
|
||||
&self,
|
||||
bucket: Option<&str>,
|
||||
prefix: Option<&str>,
|
||||
client_token: Option<&str>,
|
||||
) -> Result<HealStopOutcome, AdminClientError> {
|
||||
let mut query = vec![("forceStop", "true".to_string())];
|
||||
if let Some(token) = client_token {
|
||||
query.push(("clientToken", token.to_string()));
|
||||
}
|
||||
match client_token {
|
||||
Some(_) => {
|
||||
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::Stopped(status))
|
||||
}
|
||||
None => {
|
||||
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
|
||||
Ok(HealStopOutcome::PathStopped(success))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cluster-aggregated background heal status.
|
||||
pub async fn background_heal_status(&self) -> Result<BackgroundHealStatus, AdminClientError> {
|
||||
self.get_json("/v3/background-heal/status").await
|
||||
}
|
||||
|
||||
/// Data scanner status (enabled state, freshness, runtime config).
|
||||
pub async fn scanner_status(&self) -> Result<ScannerStatus, AdminClientError> {
|
||||
self.get_json("/v3/scanner/status").await
|
||||
}
|
||||
|
||||
/// ILM expiry worker status. The payload is owned by the expiry
|
||||
/// subsystem and still evolving; returned verbatim.
|
||||
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v3/ilm/expiry/status").await
|
||||
}
|
||||
|
||||
/// Durable replacement-recovery status (admin v4). The payload is owned
|
||||
/// by the heal runtime; returned verbatim.
|
||||
pub async fn replacement_recovery_status(&self) -> Result<serde_json::Value, AdminClientError> {
|
||||
self.get_json("/v4/heal/replacement-recovery").await
|
||||
}
|
||||
|
||||
/// Signed GET returning a decoded JSON body; escape hatch for endpoints
|
||||
/// this client does not wrap yet.
|
||||
pub async fn get_json<T: for<'de> Deserialize<'de>>(&self, path: &str) -> Result<T, AdminClientError> {
|
||||
let url = self.url_for(path, &[])?;
|
||||
let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
/// Signed POST returning a decoded JSON body.
|
||||
async fn post_json<T: for<'de> Deserialize<'de>>(
|
||||
&self,
|
||||
path: &str,
|
||||
query: &[(&str, String)],
|
||||
body: Vec<u8>,
|
||||
) -> Result<T, AdminClientError> {
|
||||
let content_type = if body.is_empty() { None } else { Some("application/json") };
|
||||
let url = self.url_for(path, query)?;
|
||||
let request = self.sign_and_build(Method::POST, url, body, content_type).await?;
|
||||
self.execute(request).await
|
||||
}
|
||||
|
||||
fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result<reqwest::Url, AdminClientError> {
|
||||
let mut url = self
|
||||
.endpoint
|
||||
.join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path))
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
|
||||
if !query.is_empty() {
|
||||
let mut pairs = url.query_pairs_mut();
|
||||
for (key, value) in query {
|
||||
pairs.append_pair(key, value);
|
||||
}
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
/// Build a SigV4-signed request via the same signer the server trusts,
|
||||
/// then hand the signed headers to the HTTP client. The signature covers
|
||||
/// method, path, query, and an unsigned-payload marker — the same shape
|
||||
/// RustFS itself sends for peer admin calls.
|
||||
async fn sign_and_build(
|
||||
&self,
|
||||
method: Method,
|
||||
url: reqwest::Url,
|
||||
body: Vec<u8>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<reqwest::Request, AdminClientError> {
|
||||
let authority = match (url.host_str(), url.port_or_known_default()) {
|
||||
(Some(host), Some(port)) => format!("{host}:{port}"),
|
||||
_ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())),
|
||||
};
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(url.as_str())
|
||||
.header(http::header::HOST, &authority)
|
||||
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
builder = builder.header(http::header::CONTENT_TYPE, content_type);
|
||||
}
|
||||
let unsigned = builder
|
||||
.body(s3s::Body::empty())
|
||||
.map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?;
|
||||
let signed = rustfs_signer::sign_v4(
|
||||
unsigned,
|
||||
body.len() as i64,
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
&self.session_token,
|
||||
&self.region,
|
||||
);
|
||||
|
||||
let mut request = self
|
||||
.http
|
||||
.request(method, url)
|
||||
.body(body)
|
||||
.build()
|
||||
.map_err(AdminClientError::Transport)?;
|
||||
let headers = request.headers_mut();
|
||||
for (name, value) in signed.headers().iter() {
|
||||
// HOST is owned by the HTTP client; the signed value above was
|
||||
// built from the same URL authority, so they always agree.
|
||||
if name == http::header::HOST {
|
||||
continue;
|
||||
}
|
||||
headers.insert(name, value.clone());
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
async fn execute<T: for<'de> Deserialize<'de>>(&self, request: reqwest::Request) -> Result<T, AdminClientError> {
|
||||
let response = self.http.execute(request).await?;
|
||||
let status = response.status();
|
||||
let bytes = response.bytes().await?;
|
||||
if !status.is_success() {
|
||||
return Err(AdminClientError::HttpStatus {
|
||||
status: status.as_u16(),
|
||||
body: String::from_utf8_lossy(&bytes).into_owned(),
|
||||
});
|
||||
}
|
||||
serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode {
|
||||
message: err.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task
|
||||
/// answers with that task's status, cancelling a whole path answers with a
|
||||
/// start-success-shaped receipt.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealStopOutcome {
|
||||
Stopped(HealTaskStatus),
|
||||
PathStopped(HealStartSuccess),
|
||||
}
|
||||
|
||||
fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String {
|
||||
match (bucket, prefix) {
|
||||
(Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => {
|
||||
format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix))
|
||||
}
|
||||
(Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => {
|
||||
format!("/v3/heal/{}", percent_encode_path_segment(bucket))
|
||||
}
|
||||
_ => "/v3/heal/".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a single path segment (slashes are content, not separators, inside
|
||||
/// bucket/prefix path params).
|
||||
fn percent_encode_path_segment(segment: &str) -> String {
|
||||
let mut out = String::with_capacity(segment.len());
|
||||
for byte in segment.bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char),
|
||||
_ => out.push_str(&format!("%{byte:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
|
||||
ScannerStatus, heal_path, percent_encode_path_segment,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[test]
|
||||
fn heal_paths_cover_root_bucket_and_prefix() {
|
||||
assert_eq!(heal_path(None, None), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/");
|
||||
assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket");
|
||||
assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_segments_percent_encode_reserved_characters() {
|
||||
assert_eq!(percent_encode_path_segment("a b"), "a%20b");
|
||||
assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb");
|
||||
assert_eq!(percent_encode_path_segment("ü"), "%C3%BC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_opts_round_trip_through_the_server_wire_shape() {
|
||||
let opts = HealOpts {
|
||||
recursive: true,
|
||||
dry_run: false,
|
||||
remove: true,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: false,
|
||||
pool: Some(1),
|
||||
set: Some(2),
|
||||
};
|
||||
let wire = serde_json::to_value(&opts).unwrap();
|
||||
assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number");
|
||||
let back: HealOpts = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(back.scan_mode, HealScanMode::Deep);
|
||||
assert_eq!(back.pool, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_scan_mode_accepts_both_wire_encodings() {
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!(1)).unwrap(), HealScanMode::Normal);
|
||||
assert_eq!(serde_json::from_value::<HealScanMode>(json!("deep")).unwrap(), HealScanMode::Deep);
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!(9)).is_err());
|
||||
assert!(serde_json::from_value::<HealScanMode>(json!("sideways")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_task_status_decodes_the_server_response_shape() {
|
||||
let raw = json!({
|
||||
"summary": "finished",
|
||||
"detail": "",
|
||||
"startTime": "2026-08-17T00:00:00Z",
|
||||
"settings": {"recursive": false, "scanMode": 1},
|
||||
"items": [{
|
||||
"resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "",
|
||||
"parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1,
|
||||
"before": {"drives": []}, "after": {"drives": []}, "objectSize": 128
|
||||
}],
|
||||
"truncated": false
|
||||
});
|
||||
let status: HealTaskStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.summary, "finished");
|
||||
assert_eq!(status.items.len(), 1);
|
||||
assert_eq!(status.settings.scan_mode, HealScanMode::Normal);
|
||||
assert!(status.progress.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
|
||||
let raw = json!({
|
||||
"state": "active",
|
||||
"bitrotStartTime": "t",
|
||||
"healQueueLength": 3,
|
||||
"healActiveTasks": 1,
|
||||
"healOperations": {"queueLength": 3},
|
||||
"clusterStatusComplete": true
|
||||
});
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.state, "active");
|
||||
assert_eq!(status.heal_queue_length, 3);
|
||||
assert!(status.cluster_status_complete);
|
||||
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_defaults_freshness_to_unknown() {
|
||||
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
|
||||
let status: ScannerStatus = serde_json::from_value(raw).unwrap();
|
||||
assert_eq!(status.freshness(), "stale");
|
||||
let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap();
|
||||
assert_eq!(bare.freshness(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_endpoint_is_rejected_without_io() {
|
||||
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
|
||||
assert!(matches!(err, AdminClientError::InvalidEndpoint(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signed_requests_carry_sigv4_authorization_and_correct_target() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin")
|
||||
.expect("client builds against the test server");
|
||||
|
||||
let start: HealStartSuccess = client
|
||||
.heal_start(
|
||||
Some("bucket"),
|
||||
None,
|
||||
&HealOpts {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("signed heal start decodes");
|
||||
|
||||
assert_eq!(start.client_token, "token-1");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(!request.query.contains("forceStart"), "absent flags must not be sent");
|
||||
let auth = request.header("authorization").expect("request must be signed");
|
||||
assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}");
|
||||
assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header");
|
||||
assert_eq!(
|
||||
request.header("x-amz-content-sha256").as_deref(),
|
||||
Some("UNSIGNED-PAYLOAD"),
|
||||
"the client signs the same payload marker RustFS peer calls use"
|
||||
);
|
||||
assert_eq!(request.header("content-type").as_deref(), Some("application/json"));
|
||||
assert!(request.body.contains("\"recursive\":true"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_sends_client_token_on_the_same_path() {
|
||||
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let status = client
|
||||
.heal_status(Some("bucket"), None, "token-1")
|
||||
.await
|
||||
.expect("status decodes");
|
||||
assert_eq!(status.summary, "running");
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
|
||||
assert!(request.query.contains("clientToken=token-1"));
|
||||
assert!(!request.query.contains("forceStop"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_without_token_takes_the_path_cancel_branch() {
|
||||
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes");
|
||||
assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_)));
|
||||
let request = server.recorded();
|
||||
assert!(request.query.contains("forceStop=true"));
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
let err = client.scanner_status().await.unwrap_err();
|
||||
match err {
|
||||
AdminClientError::HttpStatus { status, body } => {
|
||||
assert_eq!(status, 403);
|
||||
assert!(body.contains("AccessDenied"));
|
||||
}
|
||||
other => panic!("expected HttpStatus, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_success_body_maps_to_a_decode_error() {
|
||||
let server = TestServer::spawn("not json", 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. }));
|
||||
}
|
||||
|
||||
/// One recorded request, parsed off the wire with the minimum needed for
|
||||
/// assertions: method, path, query, headers, body.
|
||||
#[derive(Debug, Clone)]
|
||||
struct RecordedRequest {
|
||||
method: String,
|
||||
path: String,
|
||||
query: String,
|
||||
headers: Vec<(String, String)>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl RecordedRequest {
|
||||
fn header(&self, name: &str) -> Option<String> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal HTTP/1.1 server: one canned response per connection, every
|
||||
/// request recorded behind an `Arc<Mutex>`. Deliberately dependency-free —
|
||||
/// the assertions only need the raw request bytes.
|
||||
struct TestServer {
|
||||
addr: std::net::SocketAddr,
|
||||
requests: Arc<Mutex<Vec<RecordedRequest>>>,
|
||||
}
|
||||
|
||||
impl TestServer {
|
||||
async fn spawn(response_body: &'static str, status: u16) -> Self {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind ephemeral port");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let requests: Arc<Mutex<Vec<RecordedRequest>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let recorded = requests.clone();
|
||||
tokio::spawn(async move {
|
||||
let reason = if status == 200 { "OK" } else { "Forbidden" };
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}",
|
||||
response_body.len()
|
||||
);
|
||||
// Each request is a fresh connection (connection: close); a
|
||||
// bounded loop serves every call a test makes while letting
|
||||
// the task exit instead of lingering for the whole process.
|
||||
for _ in 0..16 {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
let mut buffer = Vec::with_capacity(2048);
|
||||
let mut chunk = [0u8; 2048];
|
||||
// Read headers plus content-length body, or stop on close.
|
||||
loop {
|
||||
if let Some(end) = find_header_end(&buffer) {
|
||||
let content_length = extract_content_length(&buffer[..end]);
|
||||
if buffer.len() >= end + content_length {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let n = match stream.read(&mut chunk).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => n,
|
||||
};
|
||||
buffer.extend_from_slice(&chunk[..n]);
|
||||
if buffer.len() > 64 * 1024 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(request) = parse_request(&buffer) {
|
||||
recorded.lock().expect("recorded lock").push(request);
|
||||
}
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
let _ = stream.shutdown().await;
|
||||
}
|
||||
});
|
||||
|
||||
Self { addr, requests }
|
||||
}
|
||||
|
||||
fn recorded(&self) -> RecordedRequest {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("recorded lock")
|
||||
.last()
|
||||
.cloned()
|
||||
.expect("the client call must have produced one recorded request")
|
||||
}
|
||||
}
|
||||
|
||||
fn find_header_end(buffer: &[u8]) -> Option<usize> {
|
||||
buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4)
|
||||
}
|
||||
|
||||
fn extract_content_length(headers: &[u8]) -> usize {
|
||||
let text = String::from_utf8_lossy(headers).to_ascii_lowercase();
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("content-length:"))
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn parse_request(raw: &[u8]) -> Option<RecordedRequest> {
|
||||
let end = find_header_end(raw)?;
|
||||
let head = String::from_utf8_lossy(&raw[..end]);
|
||||
let body = String::from_utf8_lossy(&raw[end..]).into_owned();
|
||||
let mut lines = head.lines();
|
||||
let request_line = lines.next()?;
|
||||
let mut parts = request_line.split_whitespace();
|
||||
let method = parts.next()?.to_string();
|
||||
let target = parts.next()?.to_string();
|
||||
let (path, query) = match target.split_once('?') {
|
||||
Some((path, query)) => (path.to_string(), query.to_string()),
|
||||
None => (target, String::new()),
|
||||
};
|
||||
let headers = lines
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
||||
.collect();
|
||||
Some(RecordedRequest {
|
||||
method,
|
||||
path,
|
||||
query,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod client;
|
||||
pub mod group;
|
||||
pub mod heal_commands;
|
||||
pub mod health;
|
||||
@@ -25,6 +26,7 @@ pub mod trace;
|
||||
pub mod user;
|
||||
pub mod utils;
|
||||
|
||||
pub use client::*;
|
||||
pub use group::*;
|
||||
pub use info_commands::*;
|
||||
pub use policy::*;
|
||||
|
||||
@@ -43,7 +43,7 @@ pub struct ServiceTraceOpts {
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ServiceTraceOpts {
|
||||
fn trace_types(&self) -> TraceType {
|
||||
pub fn trace_types(&self) -> TraceType {
|
||||
let mut tt = TraceType::default();
|
||||
tt.set_if(self.s3, &TraceType::S3);
|
||||
tt.set_if(self.internal, &TraceType::INTERNAL);
|
||||
@@ -72,6 +72,14 @@ impl ServiceTraceOpts {
|
||||
tt
|
||||
}
|
||||
|
||||
pub fn only_errors(&self) -> bool {
|
||||
self.only_errors
|
||||
}
|
||||
|
||||
pub fn threshold(&self) -> Duration {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> {
|
||||
let query_pairs: HashMap<_, _> = uri
|
||||
.query()
|
||||
|
||||
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMUser {
|
||||
#[serde(rename = "accessKey", default)]
|
||||
pub access_key: String,
|
||||
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRGroupInfo {
|
||||
#[serde(rename = "updateReq", default)]
|
||||
pub update_req: GroupAddRemove,
|
||||
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
|
||||
pub api_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct SRIAMItem {
|
||||
#[serde(default)]
|
||||
pub r#type: String,
|
||||
|
||||
@@ -427,7 +427,6 @@ pub enum DataSource {
|
||||
/// Write triggered
|
||||
WriteTriggered,
|
||||
/// Fallback value
|
||||
#[allow(dead_code)]
|
||||
Fallback,
|
||||
}
|
||||
|
||||
@@ -603,7 +602,6 @@ impl WriteRecord {
|
||||
|
||||
/// Hybrid strategy configuration
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct HybridStrategyConfig {
|
||||
/// Scheduled update interval
|
||||
pub scheduled_update_interval: Duration,
|
||||
@@ -998,14 +996,12 @@ impl HybridCapacityManager {
|
||||
}
|
||||
|
||||
/// Get cache age
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_cache_age(&self) -> Option<Duration> {
|
||||
let cache = self.cache.read().await;
|
||||
cache.as_ref().map(|c| c.last_update.elapsed())
|
||||
}
|
||||
|
||||
/// Get write frequency (writes/minute)
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_write_frequency(&self) -> usize {
|
||||
let record = &self.write_record;
|
||||
record.recent_write_count(record.monotonic_second())
|
||||
@@ -1300,7 +1296,6 @@ pub fn get_capacity_manager() -> Arc<HybridCapacityManager> {
|
||||
/// .update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime)
|
||||
/// .await;
|
||||
/// ```
|
||||
#[allow(dead_code)]
|
||||
pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc<HybridCapacityManager> {
|
||||
Arc::new(HybridCapacityManager::new(config))
|
||||
}
|
||||
|
||||
@@ -14,8 +14,13 @@
|
||||
|
||||
use std::any::Any;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationState, ReplicationType, ReplicationWorkerOperation};
|
||||
use crate::{
|
||||
MrfOpKind, MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING_DELETE, ReplicateObjectInfo, ReplicationState,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DeletedObjectReplicationInfo {
|
||||
@@ -117,17 +122,114 @@ pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Opti
|
||||
!(is_not_found || matches!(code, Some("MethodNotAllowed" | "405")))
|
||||
}
|
||||
|
||||
/// Build the delete-replication work item for an existing delete marker or
|
||||
/// version purge discovered during a resync scan.
|
||||
pub fn resync_existing_delete_replication_info(roi: &ReplicateObjectInfo, target_arn: &str) -> DeletedObjectReplicationInfo {
|
||||
let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: dm_version_id,
|
||||
version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker: roi.delete_marker,
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: roi.bucket.clone(),
|
||||
event_type: REPLICATE_EXISTING_DELETE.to_string(),
|
||||
op_type: ReplicationType::ExistingObject,
|
||||
target_arn: target_arn.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a delete replication fully succeeded — the MRF replay acknowledges
|
||||
/// (drops) an entry exactly when this returns true.
|
||||
///
|
||||
/// The delayed purge is deliberately NOT an input: holding the outcome hostage
|
||||
/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker
|
||||
/// entry and retained them all in the durable MRF journal forever. Purge
|
||||
/// failures persist their own purge-intent entry instead
|
||||
/// (`watch_and_purge_source_delete_marker`), and replays of those entries
|
||||
/// report purge success through `purge_stale_delete_marker_targets`.
|
||||
pub fn replicate_delete_outcome(
|
||||
expected_targets: usize,
|
||||
replicated_targets: usize,
|
||||
state_persisted: bool,
|
||||
source_state_verified: bool,
|
||||
replication_status: &ReplicationStatusType,
|
||||
) -> bool {
|
||||
expected_targets > 0
|
||||
&& replicated_targets == expected_targets
|
||||
&& state_persisted
|
||||
&& source_state_verified
|
||||
&& *replication_status == ReplicationStatusType::Completed
|
||||
}
|
||||
|
||||
pub fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||
if version_id.is_nil() {
|
||||
version_purge.then(|| NULL_VERSION_ID.to_string())
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Which version a delete-marker purge should address on one target.
|
||||
///
|
||||
/// `None` means do not purge at all: the recorded mapping disagreed across the
|
||||
/// dual internal prefixes, and guessing an id could destroy a live version on
|
||||
/// the target. `Some(id)` is the exact version the target reported when it
|
||||
/// accepted the marker; falling back to a source-derived id is only correct
|
||||
/// when the target mirrors source version ids, which a generic S3 target does
|
||||
/// not.
|
||||
pub fn delete_marker_purge_version_id(
|
||||
state: Option<&ReplicationState>,
|
||||
arn: &str,
|
||||
delete_marker_version_id: Uuid,
|
||||
) -> Option<Option<String>> {
|
||||
if state.is_some_and(|state| state.target_delete_marker_version_ids_corrupt) {
|
||||
return None;
|
||||
}
|
||||
let recorded = state.and_then(|state| state.target_delete_marker_version_ids.get(arn).cloned());
|
||||
Some(match recorded {
|
||||
Some(version_id) => Some(version_id),
|
||||
None => target_delete_version_id(delete_marker_version_id, true),
|
||||
})
|
||||
}
|
||||
|
||||
/// Shape an exhausted purge intent as a marker-creation delete entry. Replay
|
||||
/// reconstructs it with `delete_marker: true`, finds the source marker gone,
|
||||
/// and funnels into the stale-marker branch of `replicate_delete_with_outcome`
|
||||
/// — which re-runs the purge without touching source state and reports purge
|
||||
/// success as the replay outcome.
|
||||
pub fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) -> MrfReplicateEntry {
|
||||
let mut entry = dobj.to_mrf_entry();
|
||||
entry.delete_marker = true;
|
||||
entry.version_id = None;
|
||||
entry.retry_count = 0;
|
||||
entry.target_arns = failed_arns;
|
||||
entry
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge,
|
||||
DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{
|
||||
MrfOpKind, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, VersionPurgeStatusType,
|
||||
MrfOpKind, NULL_VERSION_ID, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
VersionPurgeStatusType,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -328,4 +430,100 @@ mod tests {
|
||||
assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")));
|
||||
assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied")));
|
||||
}
|
||||
|
||||
/// P1-21 regression guard for the outcome formula. A fully successful
|
||||
/// delete-marker replication must acknowledge its MRF entry: the formula
|
||||
/// once carried `&& !requires_delayed_purge`, which pinned every
|
||||
/// delete-marker entry to Missed and retained the whole backlog forever.
|
||||
/// (Deterministically staging a marker-creation entry in the durable
|
||||
/// journal from e2e would require saturating the worker queues, so the
|
||||
/// formula is pinned here instead; the purge-intent replay half is pinned
|
||||
/// by the delayed-purge e2e pair.)
|
||||
#[test]
|
||||
fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() {
|
||||
assert!(
|
||||
replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed),
|
||||
"a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending"
|
||||
);
|
||||
assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() {
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: "doc.txt".to_string(),
|
||||
// A version-purge flavored source event: the entry must still
|
||||
// be reshaped as a marker-creation delete so replay funnels
|
||||
// into the stale-marker branch instead of re-running the full
|
||||
// delete replication (whose source-state stamping would fail
|
||||
// against the already-purged version).
|
||||
delete_marker: false,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
|
||||
|
||||
assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes");
|
||||
assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay");
|
||||
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
|
||||
assert_eq!(
|
||||
entry.target_arns,
|
||||
vec!["arn:a".to_string()],
|
||||
"only the targets whose purge failed may be retried"
|
||||
);
|
||||
assert_eq!(entry.retry_count, 0);
|
||||
assert_eq!(entry.bucket, "bucket-a");
|
||||
assert_eq!(entry.object, "doc.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_delete_version_id_preserves_explicit_null_purges() {
|
||||
let version_id = Uuid::new_v4();
|
||||
|
||||
assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string()));
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID));
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), false), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_purge_prefers_the_recorded_target_version() {
|
||||
let source = Uuid::new_v4();
|
||||
let arn = "arn:rustfs:replication::target:bucket";
|
||||
|
||||
// No recorded mapping: fall back to deriving from the source uuid.
|
||||
assert_eq!(delete_marker_purge_version_id(None, arn, source), Some(Some(source.to_string())));
|
||||
|
||||
// Recorded mapping wins — a generic S3 target assigns its own id, so the
|
||||
// derived one would purge the wrong version or nothing at all.
|
||||
let mut state = ReplicationState::default();
|
||||
state
|
||||
.target_delete_marker_version_ids
|
||||
.insert(arn.to_string(), "target-assigned-id".to_string());
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(Some(&state), arn, source),
|
||||
Some(Some("target-assigned-id".to_string()))
|
||||
);
|
||||
|
||||
// A mapping recorded for a different ARN must not be reused.
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(Some(&state), "arn:rustfs:replication::other:bucket", source),
|
||||
Some(Some(source.to_string()))
|
||||
);
|
||||
|
||||
// Inconsistent persisted metadata: refuse to purge rather than guess.
|
||||
let mut corrupt = state.clone();
|
||||
corrupt.target_delete_marker_version_ids_corrupt = true;
|
||||
assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,11 @@ use uuid::Uuid;
|
||||
pub const REPLICATION_RESET: &str = "replication-reset";
|
||||
pub const REPLICATION_STATUS: &str = "replication-status";
|
||||
|
||||
/// The S3 wire spelling of the unversioned ("null") version id. Owned here as
|
||||
/// part of the replication wire contracts; `rustfs-filemeta` keeps its own
|
||||
/// copy of the same literal (the crates are intentionally independent).
|
||||
pub const NULL_VERSION_ID: &str = "null";
|
||||
|
||||
// ReplicateQueued - replication being queued trail
|
||||
pub const REPLICATE_QUEUED: &str = "replicate:queue";
|
||||
|
||||
|
||||
@@ -37,11 +37,12 @@ pub use config::{
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
pub use delete::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge,
|
||||
DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
|
||||
resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
pub use filemeta::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
|
||||
REPLICATE_INCOMING_DELETE, REPLICATE_MRF, REPLICATE_QUEUED, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision,
|
||||
ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision,
|
||||
@@ -58,8 +59,9 @@ pub use multipart::{
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan,
|
||||
};
|
||||
pub use object::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, content_matches_by_etag, replication_action_for_target,
|
||||
replication_etags_match, target_is_newer_than_source_null_version,
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag,
|
||||
is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
|
||||
ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
@@ -76,7 +78,7 @@ pub use queue::{
|
||||
pub use resync::{
|
||||
BucketReplicationResyncStatus, Error, RESYNC_FILE_MAX_BYTES, Result, ResyncOpts, ResyncStatusType,
|
||||
TargetReplicationResyncStatus, decode_resync_file, encode_resync_file, is_version_id_mismatch, resync_state_accepts_update,
|
||||
sanitize_resync_error_detail, should_auto_resume_resync, should_count_head_proxy_failure,
|
||||
resync_status_duration, sanitize_resync_error_detail, should_auto_resume_resync, should_count_head_proxy_failure,
|
||||
};
|
||||
pub use rule::ReplicationRuleExt;
|
||||
pub use runtime::{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user