mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 18:12:14 +00:00
Compare commits
53 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c568a54797 | |||
| 67904a6c18 | |||
| 2e5cef513f | |||
| 2d4f77fd3b | |||
| 920705417c | |||
| b097c94c59 | |||
| 3991a1d73c | |||
| 422e0ad768 | |||
| 3a6212f597 | |||
| ba964c82c7 | |||
| d77439929c | |||
| abc5f2e818 | |||
| 719c0d6ef0 | |||
| 83f3a7320d | |||
| 2ed28f9c5f | |||
| 0247c48ce0 | |||
| 88fa3877c1 | |||
| 2dea4a9acf | |||
| 94ee597721 | |||
| 962c11e6db | |||
| ae11bcf2be | |||
| 09157485aa | |||
| e2257325a2 | |||
| c9397405ed | |||
| 55be5af661 | |||
| 3f20fbd77b | |||
| 3c0d315a9c | |||
| 83f21eaa64 | |||
| ec25d09495 | |||
| cc24ef173c | |||
| c195b18fb8 | |||
| d670023341 | |||
| 91597db9d2 | |||
| 225918f30e | |||
| f7c1b13c0f | |||
| f329d330df | |||
| e154e0e4a2 | |||
| f235e81755 | |||
| d42bc52f8b | |||
| d48870df97 | |||
| 453e3d0faa | |||
| b65210b1db | |||
| a7f035a8c3 | |||
| 87d97a5f48 | |||
| d9efd6b853 | |||
| 2801b2500d | |||
| 02f4dbeb68 | |||
| f7757e6437 | |||
| 1cb1b02b08 | |||
| 3fe74a5019 | |||
| 451cbc099b | |||
| cb62079ba6 | |||
| d39ffdb1cd |
@@ -10,10 +10,16 @@ never weaken a check to get green.
|
||||
|
||||
## `check_layer_dependencies.sh` — layer DAG in `rustfs/src`
|
||||
|
||||
Enforces `interface (admin, storage/ecfs, storage/s3_api) → app → infra`; no
|
||||
upward imports. Known legacy violations live in
|
||||
Enforces `composition (server, startup/init) → interface (admin,
|
||||
storage/ecfs, storage/s3_api) → app → infra`; no upward imports. Server source
|
||||
files are composition roots, while imports of their exported HTTP contracts
|
||||
are classified as interface dependencies. Known legacy violations live in
|
||||
`scripts/layer-dependency-baseline.txt`.
|
||||
|
||||
Dedicated `*_test.rs` and `tests/` modules are outside this production guard.
|
||||
Inline `#[cfg(test)]` imports remain checked under their source file's layer;
|
||||
move architecture-crossing test scaffolding into a dedicated test module.
|
||||
|
||||
- **New violation**: restructure your change so the dependency points
|
||||
downward (move the shared type/function to the lower layer).
|
||||
- **You legitimately removed a baseline entry**: run
|
||||
|
||||
@@ -173,8 +173,39 @@ jobs:
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
|
||||
- name: Run nextest tests
|
||||
env:
|
||||
# Three concurrent workspace test links saturate the self-hosted
|
||||
# runner's overlay I/O and can wedge Cargo until the 75m timeout.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
|
||||
# only after `timeout` has already TERM'd the whole cargo process
|
||||
# group, so it cannot name a wedged process. Sample system and
|
||||
# process state every 60s instead; the last samples before the
|
||||
# timeout show what was stuck (rustc, linker, build script, memory
|
||||
# pressure, ...). The log rides along in the existing artifact.
|
||||
(
|
||||
while true; do
|
||||
{
|
||||
echo "=== $(date --utc --iso-8601=seconds)"
|
||||
echo "--- load"; cat /proc/loadavg
|
||||
echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true
|
||||
echo "--- mem"; free -m
|
||||
echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h /
|
||||
echo "--- top-rss"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
|
||||
echo "--- build/test processes"
|
||||
ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
|
||||
echo "--- d-state (uninterruptible IO)"
|
||||
ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
|
||||
echo
|
||||
} >> artifacts/test-and-lint/sampler.log 2>&1 || true
|
||||
sleep 60
|
||||
done
|
||||
) &
|
||||
sampler_pid=$!
|
||||
trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
|
||||
cargo nextest run --profile ci --all --exclude e2e_test \
|
||||
@@ -188,6 +219,9 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|nextest|target/.*/deps/' || true
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
@@ -372,7 +406,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary
|
||||
run: cargo build -p rustfs --bins
|
||||
run: cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -402,7 +436,7 @@ jobs:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build debug binary with rio-v2
|
||||
run: cargo build -p rustfs --bins --features rio-v2
|
||||
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
|
||||
|
||||
- name: Upload debug binary
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
|
||||
Generated
+140
-138
File diff suppressed because it is too large
Load Diff
+55
-52
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-beta.11"
|
||||
version = "1.0.0-beta.12"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,58 +86,58 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.11" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.11" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.11" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.11" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.11" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.11" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.11" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.11" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.11" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.11" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.11" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.11" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.11" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.11" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.11" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.11" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.11" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.11" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.11" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.11" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.11" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.11" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.11" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.11" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.11" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.11" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.11" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.11" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.11" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.11" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.11" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.11" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.11" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.11" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.11" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.11" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.11" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.11" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.11" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.11" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.11" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.11" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.18" }
|
||||
mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.42" }
|
||||
async-compression = { version = "0.4.43" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.91"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -152,7 +152,7 @@ lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.11.0" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.4.2"
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
minlz = "1.2.3"
|
||||
@@ -200,7 +200,7 @@ jsonwebtoken = { version = "11.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.42" }
|
||||
rustls = { default-features = false, version = "0.23.43" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
sha1 = "0.11.0"
|
||||
@@ -256,6 +256,7 @@ hashbrown = { version = "0.17.1" }
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.189"
|
||||
@@ -265,7 +266,6 @@ memmap2 = "0.9.11"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
md5 = "0.8.1"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15" }
|
||||
netif = "0.1.6"
|
||||
@@ -284,7 +284,7 @@ reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
redis = { version = "1.4.1" }
|
||||
redis = { version = "1.5.0" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
@@ -304,6 +304,7 @@ test-case = "3.3.1"
|
||||
thiserror = "2.0.19"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.33" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
@@ -314,7 +315,9 @@ uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.18" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
+145
-33
@@ -1114,6 +1114,8 @@ pub struct ScannerLastMinute {
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_active: bool,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
@@ -2051,7 +2053,7 @@ impl Metrics {
|
||||
pub fn record_scanner_transition_failed(&self, count: u64) {
|
||||
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
|
||||
self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
|
||||
}
|
||||
}
|
||||
@@ -2336,6 +2338,21 @@ impl Metrics {
|
||||
*self.cycle_info.write().await = cycle;
|
||||
}
|
||||
|
||||
/// Publish a scanner cycle and its work-accounting baseline as one state transition.
|
||||
pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
let snapshot = self.start_scan_cycle_work();
|
||||
*current_cycle = Some(cycle);
|
||||
snapshot
|
||||
}
|
||||
|
||||
/// Publish the completed work snapshot and idle cycle state as one state transition.
|
||||
pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
|
||||
let mut current_cycle = self.cycle_info.write().await;
|
||||
self.finish_scan_cycle_work(start);
|
||||
*current_cycle = Some(cycle);
|
||||
}
|
||||
|
||||
/// Read the current cycle record.
|
||||
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
|
||||
self.cycle_info.read().await.clone()
|
||||
@@ -2464,7 +2481,7 @@ impl Metrics {
|
||||
&self.current_scan_cycle_replication_repair_work_start,
|
||||
&replication_repair_snapshot,
|
||||
);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Release);
|
||||
snapshot
|
||||
}
|
||||
|
||||
@@ -2476,11 +2493,11 @@ impl Metrics {
|
||||
self.record_scan_cycle_work(work);
|
||||
self.record_scan_cycle_source_work(&source_work);
|
||||
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2746,13 +2763,41 @@ impl Metrics {
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
|
||||
let has_cycle = if let Some(cycle) = self.get_cycle().await {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed;
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
let has_cycle = {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
if m.current_cycle_active {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair =
|
||||
self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
has_cycle
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
|
||||
@@ -2793,28 +2838,6 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
|
||||
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
|
||||
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
m.current_cycle_source_work = self.scanner_source_work_snapshots(¤t_source_work);
|
||||
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
@@ -4142,6 +4165,8 @@ mod tests {
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 7);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 3);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
||||
@@ -4158,6 +4183,8 @@ mod tests {
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(!report.current_cycle_active);
|
||||
assert_eq!(report.current_cycle, 0);
|
||||
assert_eq!(report.current_cycle_objects_scanned, 0);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 0);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
||||
@@ -4184,6 +4211,91 @@ mod tests {
|
||||
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_cycle_activity_and_cycle_state_publish_together() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let active_cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(start_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
let start = start_transition.await;
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 13,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_state = metrics.cycle_info.read().await;
|
||||
let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
|
||||
assert!(finish_transition.as_mut().poll(&mut context).is_pending());
|
||||
assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
|
||||
drop(cycle_state);
|
||||
|
||||
finish_transition.await;
|
||||
let idle = metrics.report().await;
|
||||
assert!(!idle.current_cycle_active);
|
||||
assert_eq!(idle.current_cycle, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_ten = CurrentCycle {
|
||||
current: 10,
|
||||
next: 11,
|
||||
started: Utc::now() - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
|
||||
|
||||
let paths = metrics.current_paths.write().await;
|
||||
let mut report = Box::pin(metrics.report());
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(report.as_mut().poll(&mut context).is_pending());
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
|
||||
.await;
|
||||
let cycle_eleven_start = metrics
|
||||
.start_scan_cycle_work_with_cycle(CurrentCycle {
|
||||
current: 11,
|
||||
next: 12,
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
|
||||
|
||||
drop(paths);
|
||||
let snapshot = report.await;
|
||||
|
||||
assert_eq!(snapshot.current_cycle, 10);
|
||||
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -10,6 +10,9 @@ description = "Shared concurrency contract types for RustFS - workload admission
|
||||
keywords = ["rustfs", "concurrency", "admission", "backpressure", "workers"]
|
||||
categories = ["concurrency", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Internal crates
|
||||
rustfs-io-core = { workspace = true }
|
||||
|
||||
@@ -66,6 +66,10 @@ Current guidance:
|
||||
|
||||
- `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.
|
||||
|
||||
## Distributed endpoint locality
|
||||
|
||||
- `RUSTFS_LOCAL_ENDPOINT_HOST` identifies this server's host in a distributed `RUSTFS_VOLUMES` topology without resolving every peer during startup. Set it to exactly one host, without a scheme, port, or path. It is accepted only for orchestrated URL topologies and must match at least one endpoint on the RustFS server port; invalid or unmatched values fail startup. Leave it unset to retain DNS-based locality discovery.
|
||||
|
||||
## Scanner environment aliases
|
||||
|
||||
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
||||
|
||||
@@ -131,6 +131,10 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
|
||||
/// Environment variable for server volumes.
|
||||
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
|
||||
|
||||
/// Environment variable identifying this server's host in distributed endpoint
|
||||
/// lists without relying on DNS locality discovery.
|
||||
pub const ENV_LOCAL_ENDPOINT_HOST: &str = "RUSTFS_LOCAL_ENDPOINT_HOST";
|
||||
|
||||
/// Environment variable to explicitly bypass local physical disk independence checks.
|
||||
pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
|
||||
|
||||
|
||||
@@ -158,17 +158,33 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of body-bound v2 signatures.
|
||||
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
||||
/// shared secret) — and increments
|
||||
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
|
||||
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
|
||||
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
|
||||
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
|
||||
/// immediately retry with the replay-scoped signature after a peer restart.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
|
||||
|
||||
// Compile-time invariant: mixed-version clusters must remain available until operators make the
|
||||
// observed fallback counter an explicit strictness decision.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of authenticated RPC signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
@@ -354,6 +370,12 @@ mod tests {
|
||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_scope_strict_env_name_is_stable() {
|
||||
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||
|
||||
@@ -116,6 +116,27 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Request writing the complete remote-tier version state into object metadata.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested fleet-wide confirmation for
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
|
||||
///
|
||||
/// This flag is an operational contract, not automatic capability discovery.
|
||||
/// Operators may enable it only after every node that can write or read
|
||||
/// transitioned object metadata supports the remote version-state schema and
|
||||
/// semantics. Keeping the confirmation separate makes a single-node request or
|
||||
/// a writer whose local opt-in is removed fail closed.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -617,3 +638,15 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
|
||||
|
||||
/// Default read-ahead disable concurrency threshold: 4.
|
||||
pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
mod remote_version_state_tests {
|
||||
#[test]
|
||||
fn remote_version_state_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
@@ -12,12 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use path_clean::PathClean;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
path::Path,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
@@ -334,7 +332,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
|
||||
impl SizeHistogram {
|
||||
pub fn add(&mut self, size: u64) {
|
||||
let intervals = [
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -362,7 +360,7 @@ impl SizeHistogram {
|
||||
// the sub-ranges in [1 KiB, 512 KiB).
|
||||
const ONE_MIB: u64 = 1024 * 1024;
|
||||
let intervals = [
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -506,8 +504,35 @@ pub struct ReplicationStats {
|
||||
}
|
||||
|
||||
impl ReplicationStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
replicated_size,
|
||||
failed_size,
|
||||
failed_count,
|
||||
pending_count,
|
||||
missed_threshold_size,
|
||||
after_threshold_size,
|
||||
missed_threshold_count,
|
||||
after_threshold_count,
|
||||
replicated_count,
|
||||
} = self;
|
||||
|
||||
*pending_size == 0
|
||||
&& *replicated_size == 0
|
||||
&& *failed_size == 0
|
||||
&& *failed_count == 0
|
||||
&& *pending_count == 0
|
||||
&& *missed_threshold_size == 0
|
||||
&& *after_threshold_size == 0
|
||||
&& *missed_threshold_count == 0
|
||||
&& *after_threshold_count == 0
|
||||
&& *replicated_count == 0
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
|
||||
self.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -520,16 +545,19 @@ pub struct ReplicationAllStats {
|
||||
}
|
||||
|
||||
impl ReplicationAllStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
replica_size,
|
||||
replica_count,
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
pub fn empty(&self) -> bool {
|
||||
if self.replica_size != 0 && self.replica_count != 0 {
|
||||
return false;
|
||||
}
|
||||
for v in self.targets.values() {
|
||||
if !v.empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
self.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -783,7 +811,7 @@ impl DataUsageCache {
|
||||
return Some(root);
|
||||
}
|
||||
let mut flat = self.flatten(&root);
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.empty()) {
|
||||
if flat.replication_stats.as_ref().is_some_and(ReplicationAllStats::is_empty) {
|
||||
flat.replication_stats = None;
|
||||
}
|
||||
Some(flat)
|
||||
@@ -1080,9 +1108,39 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash a path for data usage caching
|
||||
fn clean_data_usage_path(data: &str) -> String {
|
||||
let rooted = data.starts_with('/');
|
||||
let mut parts = Vec::new();
|
||||
|
||||
for part in data.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
if parts.last().is_some_and(|last| *last != "..") {
|
||||
parts.pop();
|
||||
} else if !rooted {
|
||||
parts.push(part);
|
||||
}
|
||||
}
|
||||
_ => parts.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
let clean = parts.join("/");
|
||||
match (rooted, clean.is_empty()) {
|
||||
(true, true) => "/".to_string(),
|
||||
(true, false) => format!("/{clean}"),
|
||||
(false, true) => ".".to_string(),
|
||||
(false, false) => clean,
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash a slash-separated path for data usage caching.
|
||||
///
|
||||
/// Cache identifiers are persisted and exchanged across nodes, so their
|
||||
/// normalization must not depend on the host operating system.
|
||||
pub fn hash_path(data: &str) -> DataUsageHash {
|
||||
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
|
||||
DataUsageHash(clean_data_usage_path(data))
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
@@ -1467,6 +1525,23 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
("", "."),
|
||||
(".", "."),
|
||||
("/", "/"),
|
||||
("//bucket///prefix/", "/bucket/prefix"),
|
||||
("bucket/./prefix//object", "bucket/prefix/object"),
|
||||
("bucket/a/../b", "bucket/b"),
|
||||
("../bucket/..", ".."),
|
||||
("/../../bucket", "/bucket"),
|
||||
("bucket\\prefix/object", "bucket\\prefix/object"),
|
||||
] {
|
||||
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completeness_marker_is_additive_for_legacy_named_readers() {
|
||||
let current = DataUsageInfo {
|
||||
@@ -1571,6 +1646,49 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_classifies_adjacent_boundaries_once() {
|
||||
let cases = [
|
||||
(1023, 0),
|
||||
(1024, 1),
|
||||
(64 * 1024 - 1, 1),
|
||||
(64 * 1024, 2),
|
||||
(256 * 1024 - 1, 2),
|
||||
(256 * 1024, 3),
|
||||
(512 * 1024 - 1, 3),
|
||||
(512 * 1024, 4),
|
||||
(1024 * 1024 - 1, 4),
|
||||
(1024 * 1024, 6),
|
||||
(10 * 1024 * 1024 - 1, 6),
|
||||
(10 * 1024 * 1024, 7),
|
||||
(64 * 1024 * 1024 - 1, 7),
|
||||
(64 * 1024 * 1024, 8),
|
||||
(128 * 1024 * 1024 - 1, 8),
|
||||
(128 * 1024 * 1024, 9),
|
||||
(512 * 1024 * 1024 - 1, 9),
|
||||
(512 * 1024 * 1024, 10),
|
||||
];
|
||||
|
||||
for (size, expected_bucket) in cases {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(size);
|
||||
|
||||
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
|
||||
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(1024);
|
||||
|
||||
let map = hist.to_map();
|
||||
assert_eq!(map["LESS_THAN_1024_B"], 0);
|
||||
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
@@ -1582,6 +1700,126 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
("replicated_size", |stats| stats.replicated_size = 1),
|
||||
("failed_size", |stats| stats.failed_size = 1),
|
||||
("failed_count", |stats| stats.failed_count = 1),
|
||||
("pending_count", |stats| stats.pending_count = 1),
|
||||
("missed_threshold_size", |stats| stats.missed_threshold_size = 1),
|
||||
("after_threshold_size", |stats| stats.after_threshold_size = 1),
|
||||
("missed_threshold_count", |stats| stats.missed_threshold_count = 1),
|
||||
("after_threshold_count", |stats| stats.after_threshold_count = 1),
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationStats::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_all_stats_empty_checks_aggregate_fields_independently() {
|
||||
let cases = [
|
||||
(
|
||||
"replica_size",
|
||||
ReplicationAllStats {
|
||||
replica_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"replica_count",
|
||||
ReplicationAllStats {
|
||||
replica_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
assert!(ReplicationAllStats::default().is_empty());
|
||||
for (field, stats) in cases {
|
||||
assert!(!stats.is_empty(), "{field} must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!stats.is_empty(), "a non-empty target must make aggregate replication stats non-empty");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_recursive_prunes_empty_and_preserves_pending_replication_stats() {
|
||||
let root = hash_path("bucket");
|
||||
let child = hash_path("bucket/child");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(&root, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats::default()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache
|
||||
.size_recursive("bucket")
|
||||
.expect("bucket usage should flatten")
|
||||
.replication_stats
|
||||
.is_none()
|
||||
);
|
||||
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let flattened = cache.size_recursive("bucket").expect("bucket usage should flatten");
|
||||
let replication = flattened
|
||||
.replication_stats
|
||||
.expect("pending-only replication stats must survive pruning");
|
||||
|
||||
assert_eq!(replication.targets["arn:test:pending"].pending_count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_merge_adds_missing_child() {
|
||||
let mut base = DataUsageCache::default();
|
||||
|
||||
@@ -70,7 +70,8 @@ walkdir.workspace = true
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
md5 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -24,9 +24,10 @@ mod tests {
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sha2::Sha256;
|
||||
use tracing::info;
|
||||
|
||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||
@@ -70,7 +71,9 @@ mod tests {
|
||||
}
|
||||
|
||||
fn content_md5_base64(body: &[u8]) -> String {
|
||||
let digest = md5::compute(body);
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(body);
|
||||
let digest = hasher.finalize();
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ use hyper::body::Incoming;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
@@ -827,13 +828,25 @@ fn ensure_body_growth(current: usize, added: usize) -> S3Result {
|
||||
|
||||
async fn md5_digest(body: Bytes, permit: OwnedSemaphorePermit) -> S3Result<([u8; 16], OwnedSemaphorePermit)> {
|
||||
if body.len() < 1024 * 1024 {
|
||||
return Ok((md5::compute(body).0, permit));
|
||||
return Ok((md5_bytes(body), permit));
|
||||
}
|
||||
tokio::task::spawn_blocking(move || (md5::compute(body).0, permit))
|
||||
tokio::task::spawn_blocking(move || (md5_bytes(body), permit))
|
||||
.await
|
||||
.map_err(|error| s3s::s3_error!(InternalError, "MD5 worker failed: {error}"))
|
||||
}
|
||||
|
||||
fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
|
||||
let total_bytes = state
|
||||
.total_bytes
|
||||
@@ -1005,7 +1018,7 @@ impl S3 for FakeBackend {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
format!("{:x}", md5::Digest(digest))
|
||||
hex::encode(digest)
|
||||
}
|
||||
};
|
||||
let version = ObjectVersion {
|
||||
@@ -1208,7 +1221,7 @@ impl S3 for FakeBackend {
|
||||
}
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
let e_tag = format!("{:x}", md5::Digest(digest));
|
||||
let e_tag = hex::encode(digest);
|
||||
let mut state = lock(&self.store);
|
||||
let existing_bytes = state
|
||||
.uploads
|
||||
@@ -1336,7 +1349,7 @@ impl S3 for FakeBackend {
|
||||
.collect();
|
||||
let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
|
||||
let part_count = requested.len();
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{:x}-{part_count}", md5::compute(digests)));
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
|
||||
let version = ObjectVersion {
|
||||
version_id: upload.version_id.clone(),
|
||||
body,
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
|
||||
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
//! Cross-process replay / tamper acceptance for internode NodeService RPC
|
||||
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
|
||||
//! <https://github.com/rustfs/backlog/issues/1542>).
|
||||
//!
|
||||
//! # Why this exists on top of the in-process tests
|
||||
//!
|
||||
@@ -78,6 +79,8 @@
|
||||
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
|
||||
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
|
||||
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
|
||||
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
|
||||
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
|
||||
//!
|
||||
//! Two acceptance items are deliberately left to the in-process tests. A stale
|
||||
//! timestamp cannot be forged from outside — it is inside the HMAC — so
|
||||
@@ -88,18 +91,20 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use crate::storage_api::internode_rpc_signature::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_config::{
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
};
|
||||
use rustfs_protos::canonical_make_volume_request_body;
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use tonic::{Code, Request, Status};
|
||||
use tonic::{Code, Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
@@ -115,12 +120,14 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
|
||||
/// clears authentication stops harmlessly at `find_disk`.
|
||||
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
|
||||
|
||||
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
|
||||
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
|
||||
/// asserts the header it replaces was actually present, which turns a rename
|
||||
/// into a loud failure instead of silently reducing an attack to a no-op.
|
||||
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
|
||||
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
|
||||
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
|
||||
/// rename into a loud failure instead of silently reducing an attack to a no-op.
|
||||
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
|
||||
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
|
||||
/// without its leading `/`.
|
||||
@@ -155,19 +162,28 @@ fn align_rpc_secret_with_server() {
|
||||
///
|
||||
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
|
||||
/// to other tests running in the same binary.
|
||||
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
|
||||
let mut child_env = vec![
|
||||
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
|
||||
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
|
||||
];
|
||||
child_env.extend_from_slice(extra_env);
|
||||
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
|
||||
child_env
|
||||
}
|
||||
|
||||
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
start_server_with_env(&server_env(extra_env)).await
|
||||
}
|
||||
|
||||
/// Stop the child and drop the cached gRPC channel for its address.
|
||||
///
|
||||
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
|
||||
@@ -238,12 +254,83 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
|
||||
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
|
||||
/// bytes on the wire are the ones the test chose.
|
||||
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
|
||||
call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.map(Response::into_inner)
|
||||
}
|
||||
|
||||
async fn call_make_volume_response(
|
||||
url: &str,
|
||||
request: MakeVolumeRequest,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response<MakeVolumeResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(request);
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.make_volume(rpc_request).await.map(|response| response.into_inner())
|
||||
client.make_volume(rpc_request).await
|
||||
}
|
||||
|
||||
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
});
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.ping(rpc_request).await
|
||||
}
|
||||
|
||||
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
|
||||
let challenge = Uuid::new_v4();
|
||||
headers.insert(
|
||||
BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
challenge
|
||||
}
|
||||
|
||||
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 headers must carry a timestamp")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, content_sha256, boot_epoch)
|
||||
.expect("replay-scope headers must mint with the aligned RPC secret"),
|
||||
);
|
||||
headers
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
|
||||
let request = make_volume_request("signature-e2e-epoch-bootstrap");
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.expect("v2 request with epoch challenge must clear default authentication");
|
||||
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("server must HMAC-authenticate the advertised boot epoch");
|
||||
assert_authenticated(
|
||||
Ok(response.into_inner()),
|
||||
"a v2 epoch-challenge request in the default replay-scope posture",
|
||||
);
|
||||
boot_epoch
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
|
||||
let mut headers = mint_v2_headers(audience, "Ping", None);
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_ping_response(url, headers)
|
||||
.await
|
||||
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
|
||||
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("strict replay-scope Ping must return a valid boot epoch proof")
|
||||
}
|
||||
|
||||
/// Assert a call cleared authentication.
|
||||
@@ -332,6 +419,122 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
|
||||
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
|
||||
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
||||
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let child_env = server_env(&[]);
|
||||
let mut env = start_server_with_env(&child_env).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
|
||||
let request = make_volume_request("replay-scope-e2e-once");
|
||||
let captured = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request.clone(), captured.clone()).await,
|
||||
"the first replay-scoped mutation delivery",
|
||||
);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), captured).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"the same replay-scoped mutation delivered twice",
|
||||
);
|
||||
|
||||
let transplanted =
|
||||
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), transplanted).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped Ping signature transplanted onto MakeVolume",
|
||||
);
|
||||
|
||||
let stale_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
|
||||
rustfs_protos::evict_failed_connection(&url).await;
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), stale_epoch).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped signature captured before the receiving process restart",
|
||||
);
|
||||
|
||||
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
|
||||
let fresh_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
restarted_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, fresh_epoch).await,
|
||||
"a replay-scoped mutation signed with the replacement process epoch",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
|
||||
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
||||
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
|
||||
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
|
||||
assert_rejected(
|
||||
call_make_volume(
|
||||
&url,
|
||||
v2_request.clone(),
|
||||
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
|
||||
)
|
||||
.await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a v2 mutation after replay-scope strictness is enabled",
|
||||
);
|
||||
|
||||
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
|
||||
let request = make_volume_request("replay-scope-e2e-strict-v3");
|
||||
let replay_scoped = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, replay_scoped).await,
|
||||
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Baseline: correctly signed mutations are accepted, both with and without a
|
||||
/// body digest.
|
||||
///
|
||||
|
||||
@@ -30,6 +30,7 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -68,7 +69,9 @@ pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
|
||||
}
|
||||
|
||||
pub fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
BASE64.encode(md5::compute(key).0)
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
BASE64.encode(hasher.finalize())
|
||||
}
|
||||
|
||||
pub async fn kms_admin_request(
|
||||
|
||||
@@ -25,12 +25,18 @@ use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::Engine;
|
||||
use md5::compute;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Test encryption of zero-byte files (empty files)
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -294,7 +300,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
info!("🔍 Testing invalid SSE-C key length");
|
||||
let invalid_short_key = "short"; // Too short
|
||||
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
|
||||
let invalid_key_md5 = format!("{:x}", compute(invalid_short_key));
|
||||
let invalid_key_md5 = md5_hex(invalid_short_key);
|
||||
|
||||
let invalid_key_result = s3_client
|
||||
.put_object()
|
||||
|
||||
@@ -26,6 +26,7 @@ use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
@@ -50,7 +51,15 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
}
|
||||
|
||||
fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(md5::compute(key).0)
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
|
||||
@@ -5664,7 +5673,7 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let archive = make_tar(&[("alpha.txt", b"alpha-body")], &[]).await;
|
||||
let expected_etag = format!("\"{:x}\"", md5::compute(&archive));
|
||||
let expected_etag = format!("\"{}\"", md5_hex(&archive));
|
||||
|
||||
let response = client
|
||||
.put_object()
|
||||
|
||||
@@ -21,18 +21,22 @@
|
||||
//! function, never as an S3 event sink.
|
||||
//!
|
||||
//! Coverage:
|
||||
//! * PUT / multipart-complete / DELETE each deliver one event with the correct
|
||||
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
|
||||
//! eventName, bucket, key, versionId and eTag.
|
||||
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
|
||||
//! * an event queued while the target endpoint is unreachable is redelivered
|
||||
//! from the on-disk store once the endpoint recovers (store-and-forward).
|
||||
//! * responseElements and the S3 response use the canonical request ID while
|
||||
//! requestParameters preserve a conflicting client-supplied value.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use local_ip_address::local_ip;
|
||||
@@ -40,10 +44,12 @@ use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Once,
|
||||
@@ -63,6 +69,8 @@ type BoxError = Box<dyn Error + Send + Sync>;
|
||||
/// for target lookup (see `process_queue_configurations`), so the region is
|
||||
/// nominal, but it must be present for `ARN::parse` to succeed.
|
||||
const NOTIFY_REGION: &str = "us-east-1";
|
||||
const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
|
||||
const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
|
||||
|
||||
/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
|
||||
/// so the ARN a notification rule references is
|
||||
@@ -579,6 +587,36 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
|
||||
value.map(|e| e.trim_matches('"').to_string())
|
||||
}
|
||||
|
||||
fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
|
||||
assert_eq!(
|
||||
record["requestParameters"][REQUEST_ID_HEADER].as_str(),
|
||||
Some(CLIENT_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(CLIENT_AMZ_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(server_request_id),
|
||||
"notification response elements should use the canonical request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
|
||||
assert!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].is_null(),
|
||||
"notification request parameters must not invent a client request header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(request_id),
|
||||
"notification response elements should match the S3 response request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -671,8 +709,17 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
.bucket(bucket)
|
||||
.key(put_key)
|
||||
.body(ByteStream::from_static(b"peri-1 put body"))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
|
||||
request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
|
||||
assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
|
||||
assert_ne!(put_request_id, CLIENT_REQUEST_ID);
|
||||
assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
|
||||
let put_version = put
|
||||
.version_id()
|
||||
.ok_or("PUT response missing versionId (versioning not enabled?)")?;
|
||||
@@ -692,6 +739,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"record eventName: {record}"
|
||||
);
|
||||
assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
|
||||
assert_conflicting_request_id_correlation(record, &put_request_id);
|
||||
assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
|
||||
assert_eq!(
|
||||
trimmed_etag(object["eTag"].as_str()),
|
||||
@@ -744,6 +792,35 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"multipart eTag in event: {mp_record}"
|
||||
);
|
||||
|
||||
// --- Snowball extract: direct notification path keeps response correlation
|
||||
let snowball_key = "uploads/snowball.dat";
|
||||
let snowball_body = b"snowball notification body";
|
||||
let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut archive_header = tokio_tar::Header::new_gnu();
|
||||
archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
|
||||
archive_header.set_mode(0o644);
|
||||
archive_header.set_cksum();
|
||||
archive_builder
|
||||
.append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
|
||||
.await?;
|
||||
let archive = archive_builder.into_inner().await?.into_inner();
|
||||
let snowball = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("snowball-fixture.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
|
||||
|
||||
let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
let snowball_record = &snowball_event["Records"][0];
|
||||
assert_generated_request_id_correlation(snowball_record, snowball_request_id);
|
||||
|
||||
// --- Filter: non-matching prefix and suffix must never be delivered ------
|
||||
let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
|
||||
let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
|
||||
@@ -772,7 +849,32 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
|
||||
);
|
||||
|
||||
// --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
|
||||
// --- DeleteObjects: direct notification path keeps response correlation --
|
||||
let delete_many_key = "uploads/delete-many.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(delete_many_key)
|
||||
.body(ByteStream::from_static(b"delete objects notification body"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
|
||||
let delete_many = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
|
||||
let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
|
||||
|
||||
// --- DeleteObject on a versioned bucket: delete-marker version ----------
|
||||
let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
|
||||
let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
let removed_record = &removed["Records"][0];
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
||||
use tonic::Request;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
|
||||
/// gRPC lock client without authentication for testing
|
||||
/// Similar to RemoteClient but uses no_auth client
|
||||
@@ -42,7 +42,7 @@ impl GrpcLockClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
|
||||
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
node_service_time_out_client_no_auth(&self.addr)
|
||||
|
||||
@@ -23,7 +23,8 @@ use rustfs_protos::{
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
node_service_server::NodeService,
|
||||
SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -104,6 +105,27 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRenewRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseReleaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
|
||||
let request = request.into_inner();
|
||||
let args: LockRequest = match serde_json::from_str(&request.args) {
|
||||
|
||||
@@ -95,6 +95,7 @@ const MANUAL_ASYNC_PARALLEL_OBJECTS: usize = 64;
|
||||
const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
|
||||
const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
|
||||
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
|
||||
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
|
||||
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
|
||||
@@ -1684,8 +1685,15 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
|
||||
@@ -3237,6 +3237,116 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete-marker version"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
.any(|entry| !entry.delete_marker && entry.version_id == retained_version_id),
|
||||
"target removed the retained object version while purging the delete marker: {target_state:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_disabled_version_delete_preserves_target_versions_issue_5442() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-no-version-delete-src";
|
||||
let target_bucket = "replication-no-version-delete-dst";
|
||||
let object_key = "retained-versions.txt";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication_with_delete_statuses(&source_env, source_bucket, &target_arn, "Enabled", None).await?;
|
||||
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.body(ByteStream::from_static(b"permanent delete replication disabled"))
|
||||
.send()
|
||||
.await?;
|
||||
let object_version_id = put.version_id().ok_or("source PUT omitted version ID")?.to_string();
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
|
||||
let delete = source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let delete_marker_version_id = delete
|
||||
.version_id()
|
||||
.ok_or("source DELETE omitted marker version ID")?
|
||||
.to_string();
|
||||
assert_eq!(delete.delete_marker(), Some(true));
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
|
||||
let expected_target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert_eq!(expected_target_state.len(), 2);
|
||||
assert!(
|
||||
expected_target_state
|
||||
.iter()
|
||||
.any(|entry| !entry.delete_marker && entry.version_id == object_version_id)
|
||||
);
|
||||
assert!(
|
||||
expected_target_state
|
||||
.iter()
|
||||
.any(|entry| entry.delete_marker && entry.version_id == delete_marker_version_id)
|
||||
);
|
||||
|
||||
for version_id in [&object_version_id, &delete_marker_version_id] {
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert!(list_replication_state(&source_client, source_bucket).await?.is_empty());
|
||||
|
||||
let observation_deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert_eq!(
|
||||
target_state, expected_target_state,
|
||||
"disabled permanent-delete replication changed target versions"
|
||||
);
|
||||
if tokio::time::Instant::now() >= observation_deadline {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -102,10 +103,12 @@ impl Intercept for ResponseHeaderCapture {
|
||||
|
||||
fn customer_key(byte: u8) -> CustomerKey {
|
||||
let raw = [byte; 32];
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(raw);
|
||||
CustomerKey {
|
||||
raw: String::from_utf8_lossy(&raw).into_owned(),
|
||||
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,12 @@
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
|
||||
@@ -30,7 +33,7 @@ pub(crate) mod node_interact {
|
||||
}
|
||||
|
||||
pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
@@ -40,7 +43,8 @@ pub(crate) mod grpc_lock {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod internode_rpc_signature {
|
||||
pub(crate) use super::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -105,6 +105,7 @@ tempfile.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||
hostname.workspace = true
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
|
||||
@@ -153,11 +154,18 @@ metrics = { workspace = true }
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
|
||||
# dispatcher to keep tracing's process-global callsite-interest cache honest.
|
||||
tracing-core = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
|
||||
@@ -67,6 +67,14 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod evaluator {
|
||||
pub use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
}
|
||||
@@ -122,13 +130,13 @@ pub mod bucket {
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -305,7 +313,8 @@ pub mod disk {
|
||||
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -376,6 +385,7 @@ pub mod metrics {
|
||||
pub mod notification {
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -406,13 +416,15 @@ pub mod rio {
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
|
||||
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1278,6 +1278,12 @@ fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOpti
|
||||
fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
if opts.replication_request && opts.replication_delete_marker {
|
||||
None
|
||||
} else if opts.replication_request
|
||||
&& version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version_id| Uuid::parse_str(version_id).is_ok_and(|version_id| version_id.is_nil()))
|
||||
{
|
||||
Some("null".to_string())
|
||||
} else {
|
||||
version_id
|
||||
}
|
||||
@@ -2259,6 +2265,12 @@ mod tests {
|
||||
assert_eq!(got.as_deref(), Some(vid.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_version_purge_sends_s3_null_version_id() {
|
||||
let got = resolve_delete_api_version_id(Some(Uuid::nil().to_string()), &remove_opts(true, false));
|
||||
assert_eq!(got.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_propagation_omits_versionid_query_param() {
|
||||
// Propagating a delete-marker CREATION (delete_marker=true): the target
|
||||
|
||||
@@ -8974,7 +8974,7 @@ mod tests {
|
||||
.await
|
||||
.expect("first worker result should persist");
|
||||
assert_eq!(first.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(first.report.transition_completed, 1);
|
||||
assert_eq!(first.report.transition_completed, 0);
|
||||
assert_eq!(first.report.transition_failed, 0);
|
||||
|
||||
let duplicate = record_manual_transition_worker_result(
|
||||
@@ -8987,11 +8987,11 @@ mod tests {
|
||||
.await
|
||||
.expect("duplicate worker result should be idempotent");
|
||||
assert_eq!(duplicate.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(duplicate.report.transition_completed, 1);
|
||||
assert_eq!(duplicate.report.transition_completed, 0);
|
||||
assert_eq!(duplicate.report.transition_failed, 0);
|
||||
|
||||
let second_key = manual_transition_worker_result_task_key(&bucket, "logs/b", None);
|
||||
let final_record = record_manual_transition_worker_result(
|
||||
let pending_record = record_manual_transition_worker_result(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&second_key,
|
||||
@@ -9000,7 +9000,14 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("second distinct worker result should persist");
|
||||
assert_eq!(pending_record.state, ManualTransitionJobState::Running);
|
||||
assert_eq!(pending_record.report.transition_completed, 0);
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record =
|
||||
reconcile_manual_transition_worker_results(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker result journal should reconcile");
|
||||
assert_eq!(final_record.state, ManualTransitionJobState::Partial);
|
||||
assert_eq!(final_record.report.transition_completed, 1);
|
||||
assert_eq!(final_record.report.transition_failed, 1);
|
||||
@@ -9035,7 +9042,7 @@ mod tests {
|
||||
.expect("worker result job record should save");
|
||||
|
||||
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
|
||||
let final_record = record_manual_transition_worker_result_with_reason(
|
||||
let pending_record = record_manual_transition_worker_result_with_reason(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&task_key,
|
||||
@@ -9045,7 +9052,12 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("worker result with failure reason should persist");
|
||||
assert!(pending_record.report.tier_failure_by_reason.is_empty());
|
||||
assert_eq!(pending_record.report.transition_failed, 0);
|
||||
|
||||
let final_record = reconcile_manual_transition_worker_results(ecstore, job_id, ManualTransitionQueueSnapshot::default())
|
||||
.await
|
||||
.expect("worker failure reason should reconcile");
|
||||
assert_eq!(
|
||||
final_record
|
||||
.report
|
||||
@@ -11306,23 +11318,25 @@ mod tests {
|
||||
|
||||
// Distinct payloads with distinct sizes: a mixed-generation reassembly
|
||||
// would produce bytes matching none of them (or fail the read outright).
|
||||
let candidates: Vec<Vec<u8>> = (0..3)
|
||||
let candidates: Vec<Vec<u8>> = (0..2)
|
||||
.map(|g| {
|
||||
let len = 4096 + g * 512;
|
||||
vec![b'a' + g as u8; len]
|
||||
})
|
||||
.collect();
|
||||
|
||||
let commit_barrier = MultipartCommitBarrier::install(&bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
let start = Arc::new(tokio::sync::Barrier::new(candidates.len() + 1));
|
||||
let commit_barrier = MultipartCommitBarrier::install_for_arrivals(
|
||||
&bucket,
|
||||
object,
|
||||
MultipartCommitPause::PutPartBeforeLockAcquire,
|
||||
candidates.len(),
|
||||
);
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
for payload in candidates.iter().cloned() {
|
||||
let store = ecstore.clone();
|
||||
let bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let start = Arc::clone(&start);
|
||||
tasks.spawn(async move {
|
||||
start.wait().await;
|
||||
let mut data = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
|
||||
@@ -11330,11 +11344,10 @@ mod tests {
|
||||
.map(|info| (info, payload))
|
||||
});
|
||||
}
|
||||
start.wait().await;
|
||||
|
||||
// The first writer holds the uploadId commit lock while the other
|
||||
// resends reach the same critical section. Releasing it proves the
|
||||
// handoff without depending on saturated CI disk latency.
|
||||
// Both writers finish streaming before racing for the uploadId commit
|
||||
// lock. Two generations are sufficient to exercise the mixed-shard
|
||||
// hazard, while each waiter sits behind at most one cross-disk rename.
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release();
|
||||
|
||||
|
||||
@@ -1646,40 +1646,14 @@ pub async fn record_manual_transition_worker_result_with_reason(
|
||||
job_id: Uuid,
|
||||
task_key: &str,
|
||||
result: ManualTransitionWorkerResult,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
_queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
|
||||
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
||||
return load_manual_transition_job_record(api, job_id).await;
|
||||
}
|
||||
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
if record.is_terminal() {
|
||||
return Ok(record);
|
||||
}
|
||||
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
delete_manual_transition_scope_admission_if_current(
|
||||
api.clone(),
|
||||
&record.scope_key,
|
||||
record.job_id,
|
||||
record.lease_id,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
renew_manual_transition_scope_admission_from_job(api, &record).await?;
|
||||
}
|
||||
return Ok(record);
|
||||
}
|
||||
Err(Error::PreconditionFailed) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(Error::PreconditionFailed)
|
||||
load_manual_transition_job_record(api, job_id).await
|
||||
}
|
||||
|
||||
pub async fn renew_manual_transition_job_lease(
|
||||
|
||||
@@ -23,6 +23,7 @@ use uuid::Uuid;
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
Retained,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
|
||||
fn from(value: TransitionCandidateProbe) -> Self {
|
||||
match value {
|
||||
TransitionCandidateProbe::Missing => Self::Missing,
|
||||
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
|
||||
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
|
||||
TransitionCandidateProbe::Unsupported => Self::Unsupported,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorStatus {
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub tier_name: String,
|
||||
pub remote_object: String,
|
||||
pub not_after_unix_nanos: i64,
|
||||
pub probe: TransitionOperatorProbe,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorDeleteResult {
|
||||
pub status: TransitionOperatorStatus,
|
||||
pub journal_observed_after_delete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionOperatorError {
|
||||
#[error("transition transaction was not found")]
|
||||
NotFound,
|
||||
#[error("transition transaction is still inside its active ownership window")]
|
||||
NotExpired,
|
||||
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
|
||||
InvalidState(TransitionTransactionState),
|
||||
#[error("an exact non-empty remote version is required")]
|
||||
RemoteVersionRequired,
|
||||
#[error("remote candidate is not proven missing: {0:?}")]
|
||||
CandidateNotMissing(TransitionOperatorProbe),
|
||||
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
|
||||
CandidateVersionMismatch {
|
||||
expected: String,
|
||||
actual: TransitionOperatorProbe,
|
||||
},
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
Remote(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
transaction
|
||||
.validate()
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
|
||||
return Err(TransitionOperatorError::InvalidState(transaction.state));
|
||||
}
|
||||
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
|
||||
return Err(TransitionOperatorError::NotExpired);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_operator_reconcile_transaction(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionTransaction> {
|
||||
match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(transaction) => Ok(transaction),
|
||||
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => Err(TransitionOperatorError::Store(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn operator_probe_transition_candidate(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> TransitionOperatorResult<TransitionOperatorProbe> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionOperatorStatus> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api, &transaction).await?;
|
||||
Ok(TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_transition_candidate_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
remote_version_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(TransitionOperatorError::RemoteVersionRequired);
|
||||
}
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let before_delete_probe = lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
|
||||
return Err(TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected: remote_version_id.to_string(),
|
||||
actual: before_delete_probe,
|
||||
});
|
||||
}
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(_) => true,
|
||||
Err(Error::ConfigNotFound) => false,
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
Ok(TransitionOperatorDeleteResult {
|
||||
status: TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
},
|
||||
journal_observed_after_delete,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finalize_missing_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
if probe != TransitionOperatorProbe::Missing {
|
||||
return Err(TransitionOperatorError::CandidateNotMissing(probe));
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
|
||||
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
|
||||
TransitionTransaction::decode(transaction_id, data)
|
||||
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match lease
|
||||
.probe_transition_candidate(&transaction.remote_object)
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
@@ -1097,6 +1291,27 @@ mod tests {
|
||||
.expect("upload state change should succeed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
|
||||
let mut transaction = new_transaction();
|
||||
let active_deadline = transaction.not_after_unix_nanos;
|
||||
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
|
||||
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
|
||||
));
|
||||
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("unknown upload outcome should be recorded");
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
|
||||
Err(TransitionOperatorError::NotExpired)
|
||||
));
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
|
||||
.expect("expired unknown upload outcome should be eligible");
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, delete_replication_target_arns,
|
||||
replication_target_arns, should_remove_replication_target, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -28,7 +28,7 @@ use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONE
|
||||
use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, resync_target_for_object,
|
||||
delete_replication_object_opts, delete_replication_version_id, resync_target_for_object,
|
||||
};
|
||||
use super::replication_storage_boundary::{ObjectInfo, ObjectOptions, ObjectToDelete, object_to_delete_for_replication};
|
||||
use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
@@ -73,7 +73,7 @@ impl ReplicationConfig {
|
||||
if oi.delete_marker {
|
||||
let opts = ObjectOpts {
|
||||
name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
version_id: delete_replication_version_id(oi.delete_marker, oi.version_id, !oi.version_purge_status.is_empty()),
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
existing_object: true,
|
||||
@@ -299,8 +299,14 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicationRule, ReplicationRuleStatus,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
@@ -360,4 +366,56 @@ mod tests {
|
||||
assert!(options.is_replication_request());
|
||||
assert_eq!(options.user_tags(), "env=prod");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resync_distinguishes_delete_marker_creation_from_version_purge() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
endpoint: "target.example".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
let marker = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
delete_marker: true,
|
||||
version_id: Some(marker_version_id),
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let creation = config
|
||||
.resync(marker.clone(), ReplicateDecision::default(), &HashMap::new())
|
||||
.await;
|
||||
assert!(creation.targets.contains_key(arn));
|
||||
|
||||
let purge = config
|
||||
.resync(
|
||||
ObjectInfo {
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..marker
|
||||
},
|
||||
ReplicateDecision::default(),
|
||||
&HashMap::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(purge.targets.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_parts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject,
|
||||
delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object,
|
||||
should_retry_delete_marker_purge,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
};
|
||||
|
||||
@@ -67,6 +67,7 @@ const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
|
||||
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE: &str = "replication_mrf_queue_unavailable";
|
||||
const EVENT_REPLICATION_MRF_ENTRY_SKIPPED: &str = "replication_mrf_entry_skipped";
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -660,6 +661,21 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
for entry in entries.iter() {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
let Some(delete_parts) = entry.delete_parts_for_replay() else {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_MRF_ENTRY_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
reason = "invalid_delete_version_ids",
|
||||
"Skipped invalid persisted replication delete"
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let version_purge_id = delete_parts
|
||||
.version_id
|
||||
.or_else(|| delete_parts.delete_marker_version_id.filter(|_| !delete_parts.delete_marker));
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
@@ -674,15 +690,20 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
version_id: version_purge_id,
|
||||
delete_marker: delete_parts.delete_marker,
|
||||
replication_status: if entry.replica {
|
||||
ReplicationStatusType::Replica
|
||||
} else {
|
||||
ReplicationStatusType::Empty
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = check_replicate_delete(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
version_id: version_purge_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
@@ -708,9 +729,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
version_id: delete_parts.version_id,
|
||||
delete_marker_version_id: delete_parts.delete_marker_version_id,
|
||||
delete_marker: delete_parts.delete_marker,
|
||||
delete_marker_mtime,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
@@ -2333,6 +2354,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
let second = MrfReplicateEntry {
|
||||
@@ -2446,6 +2468,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
@@ -2479,6 +2502,7 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(dm_vid),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: Some(mtime_nanos),
|
||||
};
|
||||
|
||||
@@ -2512,6 +2536,7 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
@@ -2540,6 +2565,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
@@ -2551,6 +2577,7 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(del_dm_vid),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
];
|
||||
@@ -2580,6 +2607,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(obj_entry.op, MrfOpKind::Object);
|
||||
@@ -2594,6 +2622,7 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(del_entry.op, MrfOpKind::Delete);
|
||||
@@ -2609,6 +2638,7 @@ mod tests {
|
||||
op: MrfOpKind::default(),
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
|
||||
@@ -2672,6 +2702,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}];
|
||||
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
||||
@@ -2702,6 +2733,7 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}])
|
||||
.expect("invalid persisted entry should still encode for boundary testing");
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::replication_bandwidth_boundary;
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _, delete_replication_target_arns};
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
@@ -29,9 +29,10 @@ use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_msgp_boundary::ReplicationMsgpCodec;
|
||||
use super::replication_object_config::{ReplicationConfig, check_replicate_delete, get_replication_config, must_replicate};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, heal_uses_delete_replication_path,
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_replication_parts, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, should_retry_delete_marker_purge,
|
||||
version_purge_target_missing,
|
||||
};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -49,7 +50,7 @@ use super::replication_target_boundary::{
|
||||
PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore, TargetClient, replication_action_for_target_head,
|
||||
replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options,
|
||||
replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size,
|
||||
replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
replication_put_object_options, replication_target_head_is_newer_null_version, replication_target_version_id,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -70,9 +71,8 @@ use rustfs_utils::http::{
|
||||
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
@@ -97,6 +97,8 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const ERR_REPLICATION_METADATA_COPY_UNSUPPORTED: &str = "metadata-only replication is not implemented";
|
||||
const ERR_VERSION_PURGE_TARGET_STILL_EXISTS: &str = "target version still exists after replication purge";
|
||||
const ERR_VERSION_PURGE_MISSING_VERSION_ID: &str = "version purge record is missing version ID";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -786,19 +788,46 @@ impl ReplicationResyncer {
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
let Some(parts) =
|
||||
delete_replication_parts(roi.delete_marker, roi.version_id, !roi.version_purge_status.is_empty())
|
||||
else {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_OBJECT_PROCESSED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket_name,
|
||||
object = %roi.name,
|
||||
reason = "version_purge_missing_version_id",
|
||||
"Failed to process resync version purge"
|
||||
);
|
||||
let status = TargetReplicationResyncStatus {
|
||||
bucket: roi.bucket.clone(),
|
||||
object: roi.name.clone(),
|
||||
failed_count: 1,
|
||||
error: Some(ERR_VERSION_PURGE_MISSING_VERSION_ID.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(err) = results_tx.send(status).await {
|
||||
error!(
|
||||
event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket_name,
|
||||
reason = "status_channel_send_failed",
|
||||
error = %err,
|
||||
"Failed to send resync status"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
};
|
||||
|
||||
let doi = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: dm_version_id,
|
||||
version_id,
|
||||
delete_marker_version_id: parts.delete_marker_version_id,
|
||||
version_id: parts.version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker: roi.delete_marker,
|
||||
delete_marker: parts.delete_marker,
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -821,21 +850,41 @@ impl ReplicationResyncer {
|
||||
};
|
||||
|
||||
let reset_id = target_client.reset_id.clone();
|
||||
let is_version_purge = !roi.version_purge_status.is_empty();
|
||||
|
||||
let head_result = head_object_with_proxy_stats(
|
||||
&bucket_name,
|
||||
target_client.as_ref(),
|
||||
&target_client.bucket,
|
||||
&roi.name,
|
||||
roi.version_id.map(|v| v.to_string()),
|
||||
replication_target_version_id(roi.version_id, is_version_purge),
|
||||
)
|
||||
.await;
|
||||
let (size, err) = match head_result {
|
||||
Ok(_) if is_version_purge => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(ERR_VERSION_PURGE_TARGET_STILL_EXISTS.to_string()))
|
||||
}
|
||||
Ok(_) => {
|
||||
st.replicated_count += 1;
|
||||
st.replicated_size += roi.size;
|
||||
(roi.size, None)
|
||||
}
|
||||
Err(err) if is_version_purge => {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|service_err| (service_err.is_not_found(), service_err.code()))
|
||||
.unwrap_or((false, None));
|
||||
let raw_status = err.raw_response().map(|response| response.status().as_u16());
|
||||
let missing = version_purge_target_missing(is_not_found, code, raw_status);
|
||||
if missing {
|
||||
st.replicated_count += 1;
|
||||
(0, None)
|
||||
} else {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&err))
|
||||
}
|
||||
}
|
||||
Err(err) if roi.delete_marker => {
|
||||
// Verifying a replicated delete marker: only a
|
||||
// definitive 404/NoSuchKey or 405/MethodNotAllowed
|
||||
@@ -852,7 +901,7 @@ impl ReplicationResyncer {
|
||||
};
|
||||
if retryable {
|
||||
st.failed_count += 1;
|
||||
(0, Some(err))
|
||||
(0, resync_target_error_detail(&err))
|
||||
} else {
|
||||
st.replicated_count += 1;
|
||||
(0, None)
|
||||
@@ -871,17 +920,17 @@ impl ReplicationResyncer {
|
||||
}
|
||||
Ok(None) => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(err))
|
||||
(0, resync_target_error_detail(&err))
|
||||
}
|
||||
Err(e2) => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(e2))
|
||||
(0, resync_target_error_detail(&e2))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(err))
|
||||
(0, resync_target_error_detail(&err))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -911,7 +960,7 @@ impl ReplicationResyncer {
|
||||
"Processed resync object"
|
||||
);
|
||||
}
|
||||
st.error = err.as_ref().and_then(resync_target_error_detail);
|
||||
st.error = err;
|
||||
|
||||
if cancel_token.is_cancelled() {
|
||||
return;
|
||||
@@ -1065,22 +1114,27 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
}
|
||||
|
||||
let dsc = if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
|
||||
check_replicate_delete(
|
||||
oi.bucket.as_str(),
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
match delete_replication_parts(oi.delete_marker, oi.version_id, !oi.version_purge_status.is_empty()) {
|
||||
Some(parts) => {
|
||||
check_replicate_delete(
|
||||
oi.bucket.as_str(),
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: parts.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => ReplicateDecision::default(),
|
||||
}
|
||||
} else {
|
||||
must_replicate(
|
||||
oi.bucket.as_str(),
|
||||
@@ -1150,7 +1204,6 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
} else {
|
||||
dobj.delete_object.version_id
|
||||
};
|
||||
|
||||
let _rcfg = match get_replication_config(&bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
@@ -1469,8 +1522,8 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
&& replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
@@ -1600,9 +1653,50 @@ async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
fn delete_marker_purge_target_arns(config: &ReplicationConfiguration, dobj: &DeletedObjectReplicationInfo) -> HashSet<String> {
|
||||
let replica = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.replica_status == ReplicationStatusType::Replica);
|
||||
|
||||
delete_replication_target_arns(config, &dobj.delete_object.object_name, replica)
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum DeleteMarkerPurgeConfig {
|
||||
Apply(HashSet<String>),
|
||||
Stop,
|
||||
Retry,
|
||||
}
|
||||
|
||||
fn delete_marker_purge_config<E>(
|
||||
result: std::result::Result<Option<ReplicationConfiguration>, E>,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
) -> DeleteMarkerPurgeConfig {
|
||||
match result {
|
||||
Ok(Some(config)) => DeleteMarkerPurgeConfig::Apply(delete_marker_purge_target_arns(&config, dobj)),
|
||||
Ok(None) => DeleteMarkerPurgeConfig::Stop,
|
||||
Err(_) => DeleteMarkerPurgeConfig::Retry,
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(
|
||||
bucket: &str,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
) -> bool {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
return true;
|
||||
};
|
||||
let marker_creation_purge_targets = if dobj.delete_object.delete_marker {
|
||||
match delete_marker_purge_config(get_replication_config(bucket).await, dobj) {
|
||||
DeleteMarkerPurgeConfig::Apply(targets) => Some(targets),
|
||||
DeleteMarkerPurgeConfig::Stop => return true,
|
||||
DeleteMarkerPurgeConfig::Retry => return false,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
@@ -1612,6 +1706,12 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
if marker_creation_purge_targets
|
||||
.as_ref()
|
||||
.is_some_and(|targets| !targets.contains(&tgt_entry.arn))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
continue;
|
||||
};
|
||||
@@ -1625,6 +1725,7 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
)
|
||||
.await;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
@@ -1851,10 +1952,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
let version_id = if let Some(version_id) = dobj.delete_object.delete_marker_version_id {
|
||||
Some(version_id)
|
||||
} else {
|
||||
dobj.delete_object.version_id.unwrap_or_default()
|
||||
dobj.delete_object.version_id
|
||||
};
|
||||
|
||||
let mut rinfo = dobj
|
||||
@@ -1889,11 +1990,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
let version_id = if version_id.is_nil() {
|
||||
None
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
};
|
||||
let version_id = replication_target_version_id(version_id, is_version_purge);
|
||||
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_with_proxy_stats(
|
||||
@@ -3143,6 +3240,10 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicaModifications, ReplicaModificationsStatus,
|
||||
ReplicationRule, ReplicationRuleAndOperator, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
@@ -3452,6 +3553,110 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_targets_follow_delete_and_replica_modification_rules() {
|
||||
fn rule(arn: &str, delete_status: &'static str) -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(delete_status),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some(arn.to_string()),
|
||||
prefix: Some("logs/".to_string()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}
|
||||
}
|
||||
|
||||
let enabled_arn = "arn:rustfs:replication:us-east-1:target:enabled";
|
||||
let disabled_arn = "arn:rustfs:replication:us-east-1:target:disabled";
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
rule(enabled_arn, DeleteReplicationStatus::ENABLED),
|
||||
rule(disabled_arn, DeleteReplicationStatus::DISABLED),
|
||||
],
|
||||
};
|
||||
let mut dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "logs/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(delete_marker_purge_target_arns(&config, &dobj), HashSet::from([enabled_arn.to_string()]));
|
||||
assert_eq!(
|
||||
delete_marker_purge_config::<()>(Ok(Some(config.clone())), &dobj),
|
||||
DeleteMarkerPurgeConfig::Apply(HashSet::from([enabled_arn.to_string()]))
|
||||
);
|
||||
|
||||
dobj.delete_object.replication_state = Some(Default::default());
|
||||
dobj.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("test replication state")
|
||||
.replica_status = ReplicationStatusType::Replica;
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
|
||||
config.rules[0].source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
assert_eq!(delete_marker_purge_target_arns(&config, &dobj), HashSet::from([enabled_arn.to_string()]));
|
||||
|
||||
config.rules[0].prefix = None;
|
||||
config.rules[0].filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
|
||||
config.rules[0].filter = Some(ReplicationRuleFilter {
|
||||
and: Some(ReplicationRuleAndOperator {
|
||||
prefix: Some("logs/".to_string()),
|
||||
tags: Some(vec![Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}]),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_config_errors_are_retryable() {
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(delete_marker_purge_config::<()>(Ok(None), &dobj), DeleteMarkerPurgeConfig::Stop);
|
||||
assert_eq!(delete_marker_purge_config::<()>(Err(()), &dobj), DeleteMarkerPurgeConfig::Retry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
|
||||
@@ -31,10 +31,13 @@ use rustfs_utils::http::{
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
@@ -305,6 +308,14 @@ pub(crate) fn replication_target_head_is_newer_null_version(object_info: &Object
|
||||
target_is_newer_than_source_null_version(&replication_source_object(object_info), &replication_target_object(target))
|
||||
}
|
||||
|
||||
pub(crate) fn replication_target_version_id(version_id: Option<Uuid>, version_purge: bool) -> Option<String> {
|
||||
match version_id {
|
||||
Some(version_id) if version_id.is_nil() && version_purge => Some("null".to_string()),
|
||||
Some(version_id) if !version_id.is_nil() => Some(version_id.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replication_delete_remove_options(
|
||||
delete_marker: bool,
|
||||
replication_mtime: Option<OffsetDateTime>,
|
||||
@@ -519,6 +530,18 @@ mod tests {
|
||||
assert!(force.replication_request);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_version_id_preserves_null_purges() {
|
||||
assert_eq!(replication_target_version_id(Some(Uuid::nil()), true).as_deref(), Some("null"));
|
||||
assert_eq!(replication_target_version_id(Some(Uuid::nil()), false), None);
|
||||
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
replication_target_version_id(Some(version_id), true).as_deref(),
|
||||
Some(version_id.to_string().as_str())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_complete_multipart_options_sets_actual_size() {
|
||||
let options = replication_complete_multipart_options("1024".to_string());
|
||||
|
||||
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
if version_id.is_empty() {
|
||||
return Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
|
||||
@@ -12,11 +12,20 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
|
||||
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
|
||||
};
|
||||
use crate::cluster::rpc::{
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::Uri;
|
||||
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
@@ -24,9 +33,19 @@ use rustfs_protos::{
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
};
|
||||
use std::{error::Error, io::ErrorKind};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
error::Error,
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{LazyLock, Mutex},
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tower::Service;
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
|
||||
|
||||
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
|
||||
pub async fn node_service_time_out_client(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
|
||||
// `_for_class` variant below (grpc-optimization P1).
|
||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
|
||||
pub async fn heal_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
|
||||
pub async fn tier_mutation_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
class: ChannelClass,
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match class {
|
||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
};
|
||||
|
||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
|
||||
pub async fn node_service_time_out_client_no_auth(
|
||||
addr: &String,
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||
}
|
||||
|
||||
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
|
||||
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
|
||||
/// old servers continue receiving precisely the metadata they understand.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ReplayScopeChannel<S> {
|
||||
inner: S,
|
||||
audience: Option<String>,
|
||||
}
|
||||
|
||||
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||
|
||||
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
impl<S> ReplayScopeChannel<S> {
|
||||
fn new(inner: S, audience: Option<String>) -> Self {
|
||||
Self { inner, audience }
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
|
||||
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
|
||||
}
|
||||
|
||||
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
|
||||
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
|
||||
epochs.insert(audience, epoch);
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
S::Error: Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
ReqBody: Send + 'static,
|
||||
ResBody: Send + 'static,
|
||||
{
|
||||
type Response = HttpResponse<ResBody>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
== Some(RPC_AUTH_VERSION_V2)
|
||||
});
|
||||
let challenge = authenticated.then(Uuid::new_v4);
|
||||
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
|
||||
// The challenge is independently HMAC-authenticated by the response proof. It is not
|
||||
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
|
||||
request.headers_mut().insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
|
||||
cached_peer_boot_epoch(audience),
|
||||
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
) {
|
||||
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
|
||||
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
|
||||
Err(error)
|
||||
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|
||||
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
|
||||
{
|
||||
debug!(error = %error, "peer boot epoch response proof was rejected")
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
}
|
||||
@@ -257,6 +377,13 @@ impl TonicInterceptor {
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
fn replay_scope_audience(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Signature(interceptor) => interceptor.audience.clone(),
|
||||
Self::NoOp(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
@@ -279,6 +406,38 @@ mod tests {
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct EpochProofService {
|
||||
audience: String,
|
||||
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
|
||||
}
|
||||
|
||||
impl Service<HttpRequest<()>> for EpochProofService {
|
||||
type Response = HttpResponse<()>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
|
||||
self.seen_headers
|
||||
.lock()
|
||||
.expect("test header capture lock must not be poisoned")
|
||||
.push(request.headers().clone());
|
||||
let challenge = tonic_boot_epoch_challenge(request.headers())
|
||||
.expect("client challenge must be syntactically valid")
|
||||
.expect("authenticated client request must carry a boot epoch challenge");
|
||||
let mut response = HttpResponse::new(());
|
||||
response.headers_mut().extend(
|
||||
tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||
.expect("test server must be able to sign an epoch proof"),
|
||||
);
|
||||
std::future::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_test_rpc_secret() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
@@ -420,6 +579,52 @@ mod tests {
|
||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "replay-scope-client-test:9000";
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
seen_headers: seen_headers.clone(),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
let make_request = || {
|
||||
let mut request = HttpRequest::builder()
|
||||
.uri("/node_service.NodeService/Ping")
|
||||
.body(())
|
||||
.expect("test RPC request must build");
|
||||
request.headers_mut().extend(
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 test headers must mint"),
|
||||
);
|
||||
request
|
||||
};
|
||||
|
||||
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
|
||||
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
|
||||
|
||||
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
|
||||
assert_eq!(headers.len(), 2);
|
||||
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
|
||||
assert!(
|
||||
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the first request must remain v2-compatible until the peer proves its epoch"
|
||||
);
|
||||
assert!(
|
||||
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the second request must carry the replay-scoped v3 signature"
|
||||
);
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
||||
//! receive process-local replay protection. See
|
||||
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
|
||||
//! requests additionally receive process-local replay protection. See
|
||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
@@ -50,13 +50,22 @@ use uuid::Uuid;
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
|
||||
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
|
||||
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
|
||||
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
|
||||
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
||||
// cannot disable replay protection by rejecting every body-bound request.
|
||||
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ReplayScope<'a> {
|
||||
audience: &'a str,
|
||||
path: &'a str,
|
||||
timestamp: &'a str,
|
||||
nonce: Uuid,
|
||||
content_sha256: &'a str,
|
||||
boot_epoch: Uuid,
|
||||
}
|
||||
|
||||
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||
for part in [
|
||||
scope.audience.as_bytes(),
|
||||
b"|",
|
||||
scope.path.as_bytes(),
|
||||
b"|POST|",
|
||||
scope.timestamp.as_bytes(),
|
||||
b"|",
|
||||
scope.nonce.as_bytes(),
|
||||
b"|",
|
||||
scope.content_sha256.as_bytes(),
|
||||
b"|",
|
||||
scope.boot_epoch.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_scope(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
update_replay_scope(&mut mac, scope);
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
|
||||
mac.update(audience.as_bytes());
|
||||
mac.update(b"|");
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(boot_epoch.as_bytes());
|
||||
}
|
||||
|
||||
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
.then_some(value)
|
||||
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
|
||||
}
|
||||
|
||||
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
|
||||
path.strip_prefix('/')
|
||||
.and_then(|path| path.split_once('/'))
|
||||
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
|
||||
}
|
||||
|
||||
/// The process-unique epoch included in every replay-scoped server verification.
|
||||
///
|
||||
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
|
||||
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
|
||||
pub fn tonic_rpc_boot_epoch() -> Uuid {
|
||||
*RPC_BOOT_EPOCH
|
||||
}
|
||||
|
||||
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
|
||||
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
|
||||
/// old servers can continue validating the same request unchanged.
|
||||
pub fn gen_tonic_replay_scope_headers(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
|
||||
let nonce = Uuid::new_v4();
|
||||
let signature = generate_replay_scope_signature(
|
||||
&get_shared_secret()?,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
|
||||
);
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
|
||||
);
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
|
||||
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
|
||||
headers
|
||||
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Verify the server boot-epoch response for a challenge generated by this client.
|
||||
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let proof = headers
|
||||
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
|
||||
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(boot_epoch)
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -443,6 +642,16 @@ pub fn set_tonic_canonical_body_digest<T>(request: &mut tonic::Request<T>, canon
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
|
||||
request: &mut tonic::Request<T>,
|
||||
) -> std::io::Result<()> {
|
||||
let canonical_body = request
|
||||
.get_ref()
|
||||
.canonical_body()
|
||||
.map_err(|_| std::io::Error::other("RPC mutation body length cannot be represented"))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body)
|
||||
}
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
@@ -466,7 +675,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a mutating disk RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
/// Verify a mutating RPC's canonical body digest with a rolling-upgrade fallback.
|
||||
///
|
||||
/// When the request carries a real (non-`UNSIGNED-PAYLOAD`) content SHA-256 it is verified exactly
|
||||
/// like [`verify_tonic_canonical_body_digest`]. The digest value is a member of the signed v2
|
||||
@@ -497,7 +706,7 @@ fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
Some(digest) if digest != UNSIGNED_PAYLOAD => verify_tonic_canonical_body_digest(request, canonical_body),
|
||||
_ => {
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): accept digestless peers during rolling upgrades. Remove after the
|
||||
// minimum supported RustFS peer version body-binds every mutating disk RPC.
|
||||
// minimum supported RustFS peer version body-binds every mutating RPC.
|
||||
if strict {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
@@ -521,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
]
|
||||
.iter()
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
|
||||
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
|
||||
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
|
||||
@@ -530,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
|
||||
*INTERNODE_RPC_SIGNATURE_STRICT
|
||||
}
|
||||
|
||||
fn internode_rpc_replay_scope_strict() -> bool {
|
||||
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
|
||||
}
|
||||
|
||||
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
if audience.is_empty() {
|
||||
return Err(std::io::Error::other("Missing RPC audience"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
|
||||
let version = headers
|
||||
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
|
||||
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
|
||||
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
|
||||
}
|
||||
let signature = headers
|
||||
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let signed_at = timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(signed_at)?;
|
||||
let nonce = headers
|
||||
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let secret = get_shared_secret()?;
|
||||
if !verify_replay_scope_signature(
|
||||
&secret,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
signature,
|
||||
) {
|
||||
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
|
||||
}
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_nonce(nonce, signed_at)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
|
||||
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
|
||||
pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
allow_replay_scope_bootstrap,
|
||||
)
|
||||
}
|
||||
|
||||
fn verify_tonic_rpc_signature_with_policy(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
signature_strict: bool,
|
||||
replay_scope_strict: bool,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if has_replay_scope_headers(headers) {
|
||||
return verify_tonic_replay_scope_signature(audience, path, headers);
|
||||
}
|
||||
|
||||
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
|
||||
// client after its peer restarts. Legacy metadata never gets this exception.
|
||||
let bootstrap = allow_replay_scope_bootstrap
|
||||
&& has_v2_auth_headers(headers)
|
||||
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
|
||||
if replay_scope_strict && !bootstrap {
|
||||
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
|
||||
}
|
||||
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
|
||||
global_internode_metrics().record_replay_scope_fallback();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
|
||||
@@ -677,11 +1015,28 @@ mod tests {
|
||||
use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_protos::{
|
||||
CanonicalMutationBody as _, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
proto_gen::node_service::{Mss, SignalServiceRequest},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
fn signal_service_request(signal: &str, sub_system: &str, dry_run: &str) -> SignalServiceRequest {
|
||||
SignalServiceRequest {
|
||||
vars: Some(Mss {
|
||||
value: HashMap::from([
|
||||
(PEER_RESTSIGNAL.to_string(), signal.to_string()),
|
||||
(PEER_RESTSUB_SYS.to_string(), sub_system.to_string()),
|
||||
(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()),
|
||||
]),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -1246,6 +1601,104 @@ mod tests {
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_binds_path_epoch_and_random_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
|
||||
"the first replay-scoped request must be accepted"
|
||||
);
|
||||
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
|
||||
.expect_err("the random replay-scope nonce must be single-use");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("a replay-scoped signature must not move to another method");
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("partial replay-scope metadata must never downgrade to v2");
|
||||
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
|
||||
|
||||
let timestamp = partial
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = partial
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
let stale_epoch = Uuid::new_v4();
|
||||
partial.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, stale_epoch)
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("a signature from a prior server boot epoch must be rejected");
|
||||
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let rejected =
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
|
||||
.expect_err("strict replay scope must reject stripped new metadata");
|
||||
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
|
||||
|
||||
headers.insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
|
||||
);
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
|
||||
.is_ok(),
|
||||
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
|
||||
let epoch =
|
||||
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
|
||||
assert_eq!(epoch, tonic_rpc_boot_epoch());
|
||||
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -1596,6 +2049,72 @@ mod tests {
|
||||
assert_eq!(stripped.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signal_service_mutation_contract_rejects_tampering_and_replay() {
|
||||
ensure_test_rpc_secret();
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "SignalService", content_sha256)
|
||||
.expect("body-bound auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers).is_ok(),
|
||||
"the first body-bound signal request must authenticate"
|
||||
);
|
||||
assert!(verify_tonic_mutation_body_digest(&request, &body).is_ok());
|
||||
|
||||
let tampered = signal_service_request("1", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let error = verify_tonic_mutation_body_digest(&request, &tampered)
|
||||
.expect_err("changing the signal must invalidate the signed digest");
|
||||
assert_eq!(error.to_string(), "RPC content SHA-256 mismatch");
|
||||
|
||||
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("reusing the signal nonce must fail");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(rpc_body_digest_fallback_counter)]
|
||||
fn signal_service_mutation_contract_preserves_rollout_fallback_and_strictness() {
|
||||
let body = signal_service_request("2", "scanner", "false")
|
||||
.canonical_body()
|
||||
.expect("small signal request should encode");
|
||||
let before = global_internode_metrics().snapshot().body_digest_fallback_total;
|
||||
let digestless = tonic::Request::new(());
|
||||
|
||||
assert!(
|
||||
verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, false).is_ok(),
|
||||
"old peers must remain compatible while the rollout gate is open"
|
||||
);
|
||||
assert_eq!(
|
||||
global_internode_metrics().snapshot().body_digest_fallback_total,
|
||||
before + 1,
|
||||
"accepted digestless signal requests must be visible in the fallback metric"
|
||||
);
|
||||
|
||||
let error = verify_tonic_mutation_body_digest_with_strictness(&digestless, &body, true)
|
||||
.expect_err("strict mode must reject a digestless signal request");
|
||||
assert_eq!(error.to_string(), "RPC mutation requires a body-bound v2 signature");
|
||||
|
||||
let mut bound = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut bound, &body).expect("canonical body digest should be attached");
|
||||
bound
|
||||
.metadata_mut()
|
||||
.as_mut()
|
||||
.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
assert!(verify_tonic_mutation_body_digest_with_strictness(&bound, &body, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_rejects_replay_after_wall_clock_regression() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -26,13 +26,18 @@ pub(crate) mod runtime_sources;
|
||||
pub use background_monitor::shutdown_background_monitors;
|
||||
pub(crate) use background_monitor::spawn_background_monitor;
|
||||
pub use client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth,
|
||||
};
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -13,10 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -50,6 +50,7 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
@@ -57,7 +58,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
io::Cursor,
|
||||
sync::{
|
||||
Arc,
|
||||
Arc, Weak,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::SystemTime,
|
||||
@@ -65,13 +66,9 @@ use std::{
|
||||
use tokio::{net::TcpStream, time::Duration};
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
pub const PEER_RESTDRY_RUN: &str = "dry-run";
|
||||
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
|
||||
@@ -224,6 +221,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
return Err(Error::other(
|
||||
"peer returned a remote version state capability for a different topology member",
|
||||
));
|
||||
}
|
||||
let server_epoch =
|
||||
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
|
||||
if server_epoch.is_nil() {
|
||||
return Err(Error::other("peer returned a nil remote version state epoch"));
|
||||
}
|
||||
Ok(server_epoch)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -235,6 +247,7 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
pub grid_host: String,
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -327,14 +340,54 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub fn new(host: XHost, grid_host: String) -> Self {
|
||||
let topology_member = host.to_string();
|
||||
Self {
|
||||
host,
|
||||
grid_host,
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
|
||||
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| url.path() != "/"
|
||||
{
|
||||
return Err(Error::other("peer grid host has an invalid URL shape"));
|
||||
}
|
||||
let url_host = url.host().ok_or_else(|| Error::other("peer grid host is missing a host"))?;
|
||||
let topology_host = match url.port() {
|
||||
Some(port) => format!("{url_host}:{port}"),
|
||||
None => url_host.to_string(),
|
||||
};
|
||||
let explicit_port = url.port();
|
||||
let name = match url_host {
|
||||
url::Host::Domain(domain) => domain.to_string(),
|
||||
url::Host::Ipv4(address) => address.to_string(),
|
||||
url::Host::Ipv6(address) if explicit_port.is_none() => format!("[{address}]"),
|
||||
url::Host::Ipv6(address) => address.to_string(),
|
||||
};
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.filter(|port| *port > 0)
|
||||
.ok_or_else(|| Error::other("peer grid host is missing a valid port"))?;
|
||||
let host = XHost {
|
||||
name,
|
||||
port,
|
||||
is_port_set: explicit_port.is_some(),
|
||||
};
|
||||
if topology_host != peer_host_port {
|
||||
return Err(Error::other("peer topology host does not match its grid URL"));
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
fn build_clients_from_slots(
|
||||
slots: Vec<(String, Option<String>, bool)>,
|
||||
) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
|
||||
@@ -348,10 +401,14 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
let client = match grid_host {
|
||||
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
|
||||
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
|
||||
Some(grid_host) => match Self::parse_topology_host(&peer_host_port, &grid_host) {
|
||||
Ok(host) => {
|
||||
let mut client = PeerRestClient::new(host, grid_host);
|
||||
client.topology_member = peer_host_port.clone();
|
||||
Some(client)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
|
||||
warn!(peer = %peer_host_port, "peer topology host parse failed while constructing peer client: {err:?}");
|
||||
None
|
||||
}
|
||||
},
|
||||
@@ -392,7 +449,7 @@ impl PeerRestClient {
|
||||
(remote, all, remote_topology_hosts)
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -413,7 +470,7 @@ impl PeerRestClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
|
||||
InterceptedService<Channel, TonicInterceptor>,
|
||||
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
@@ -434,7 +491,7 @@ impl PeerRestClient {
|
||||
|
||||
async fn get_tier_mutation_control_client(
|
||||
&self,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -500,9 +557,8 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
let grid_host = self.grid_host.clone();
|
||||
let offline = Arc::clone(&self.offline);
|
||||
let recovery_running = Arc::clone(&self.recovery_running);
|
||||
let span = Self::recovery_monitor_span(&grid_host);
|
||||
let offline = Arc::downgrade(&self.offline);
|
||||
let recovery_running = Arc::downgrade(&self.recovery_running);
|
||||
// The offline flag and its recovery are the silent half of
|
||||
// rustfs/backlog#888: log the monitor's start and its success so an
|
||||
// "offline then back" episode leaves a trace on the observing node.
|
||||
@@ -511,13 +567,34 @@ impl PeerRestClient {
|
||||
grid_host = %self.grid_host,
|
||||
"peer RPC connection marked offline after a network-like failure; starting background recovery monitor"
|
||||
);
|
||||
drop(Self::spawn_recovery_monitor(grid_host, offline, recovery_running));
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor(
|
||||
grid_host: String,
|
||||
offline: Weak<AtomicBool>,
|
||||
recovery_running: Weak<AtomicBool>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let span = Self::recovery_monitor_span(&grid_host);
|
||||
super::spawn_background_monitor(span, async move {
|
||||
let mut delay = get_drive_active_check_interval();
|
||||
let connect_timeout = get_drive_active_check_timeout();
|
||||
|
||||
for attempt in 1..=PEER_REST_RECOVERY_MAX_ATTEMPTS {
|
||||
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
|
||||
return;
|
||||
}
|
||||
if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() {
|
||||
let Some(offline) = offline.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let Some(recovery_running) = recovery_running.upgrade() else {
|
||||
return;
|
||||
};
|
||||
offline.store(false, Ordering::Release);
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
info!(
|
||||
@@ -537,8 +614,10 @@ impl PeerRestClient {
|
||||
attempts = PEER_REST_RECOVERY_MAX_ATTEMPTS,
|
||||
"peer recovery monitor reached max attempts; will retry on next request"
|
||||
);
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
});
|
||||
if let Some(recovery_running) = recovery_running.upgrade() {
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1156,14 +1235,24 @@ impl PeerRestClient {
|
||||
validate_heal_control_capability_proof(&canonical_ack, &proof)
|
||||
}
|
||||
|
||||
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadBucketMetadataRequest {
|
||||
let mut request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1183,9 +1272,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(DeleteBucketMetadataRequest {
|
||||
let mut request = Request::new(DeleteBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1205,9 +1295,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(DeletePolicyRequest {
|
||||
let mut request = Request::new(DeletePolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1227,9 +1318,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadPolicyRequest {
|
||||
let mut request = Request::new(LoadPolicyRequest {
|
||||
policy_name: policy.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1249,11 +1341,12 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadPolicyMappingRequest {
|
||||
let mut request = Request::new(LoadPolicyMappingRequest {
|
||||
user_or_group: user_or_group.to_string(),
|
||||
user_type,
|
||||
is_group,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_policy_mapping(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1273,9 +1366,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(DeleteUserRequest {
|
||||
let mut request = Request::new(DeleteUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1295,9 +1389,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(DeleteServiceAccountRequest {
|
||||
let mut request = Request::new(DeleteServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.delete_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1317,10 +1412,11 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadUserRequest {
|
||||
let mut request = Request::new(LoadUserRequest {
|
||||
access_key: access_key.to_string(),
|
||||
temp,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_user(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1340,9 +1436,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadServiceAccountRequest {
|
||||
let mut request = Request::new(LoadServiceAccountRequest {
|
||||
access_key: access_key.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_service_account(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1362,9 +1459,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadGroupRequest {
|
||||
let mut request = Request::new(LoadGroupRequest {
|
||||
group: group.to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_group(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1384,7 +1482,8 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
let mut request = Request::new(ReloadSiteReplicationConfigRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.reload_site_replication_config(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1408,9 +1507,10 @@ impl PeerRestClient {
|
||||
vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
|
||||
vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
|
||||
vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
|
||||
let request = Request::new(SignalServiceRequest {
|
||||
let mut request = Request::new(SignalServiceRequest {
|
||||
vars: Some(Mss { value: vars }),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.signal_service(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1479,7 +1579,8 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(ReloadPoolMetaRequest {});
|
||||
let mut request = Request::new(ReloadPoolMetaRequest {});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.reload_pool_meta(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1500,9 +1601,10 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(StopRebalanceRequest {
|
||||
let mut request = Request::new(StopRebalanceRequest {
|
||||
expected_rebalance_id: expected_rebalance_id.unwrap_or_default().to_string(),
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.stop_rebalance(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1523,7 +1625,8 @@ impl PeerRestClient {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
let mut request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_rebalance_meta(request).await?.into_inner();
|
||||
|
||||
@@ -1562,7 +1665,8 @@ impl PeerRestClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
let mut request = Request::new(StartDecommissionRequest { pool_indices });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.start_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1585,7 +1689,8 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
let mut request = Request::new(CancelDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.cancel_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1608,7 +1713,8 @@ impl PeerRestClient {
|
||||
let pool_index = u32::try_from(pool_index)
|
||||
.map_err(|_| Error::other(format!("decommission pool index {pool_index} exceeds RPC range")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
let mut request = Request::new(ClearDecommissionRequest { pool_index });
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.clear_decommission(request).await?.into_inner();
|
||||
if !response.success {
|
||||
@@ -1628,10 +1734,14 @@ impl PeerRestClient {
|
||||
pub async fn load_transition_tier_config(&self) -> Result<()> {
|
||||
match self.load_transition_tier_config_outcome().await {
|
||||
TierConfigReloadOutcome::Success => Ok(()),
|
||||
TierConfigReloadOutcome::TransientReconnect(err) | TierConfigReloadOutcome::TransientRetrySameChannel(err) => {
|
||||
self.finalize_result(Err(err)).await
|
||||
}
|
||||
TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
// Only a reconnect-class failure says anything about the channel.
|
||||
// `finalize_result` marks the peer offline and evicts its connection
|
||||
// whenever the message looks network-like, and a peer that answered
|
||||
// and rejected the apply can easily report one ("release RPC failed:
|
||||
// transport error"). Routing those through here would gate a healthy,
|
||||
// responding peer out of every unrelated RPC.
|
||||
TierConfigReloadOutcome::TransientReconnect(err) => self.finalize_result(Err(err)).await,
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(err) | TierConfigReloadOutcome::Terminal(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1657,6 +1767,9 @@ impl PeerRestClient {
|
||||
Err(err) => return tier_config_reload_connection_outcome(err),
|
||||
};
|
||||
let mut request = Request::new(LoadTransitionTierConfigRequest {});
|
||||
if let Err(err) = set_tonic_mutation_body_digest(&mut request) {
|
||||
return TierConfigReloadOutcome::Terminal(Error::other(err));
|
||||
}
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
|
||||
let response = match client.load_transition_tier_config(request).await {
|
||||
@@ -1710,13 +1823,24 @@ fn is_tier_config_reload_connection_failure(err: &Error) -> bool {
|
||||
message_has_network_needle(&message)
|
||||
}
|
||||
|
||||
/// Classifies a reload the peer answered but refused to apply.
|
||||
///
|
||||
/// The peer replied, so the channel is healthy and only the remote apply
|
||||
/// failed. Those failures are transient by nature: the reload reads the tier
|
||||
/// mutation intents and takes the distributed tier-config lock, both of which
|
||||
/// fail while any other node is restarting or while the lock quorum is briefly
|
||||
/// disturbed. Retiring the worker on the first such rejection leaves that peer
|
||||
/// pinned to the old configuration with nothing left to heal it, so it answers
|
||||
/// `TierNotFound` for a tier the rest of the cluster already committed until a
|
||||
/// second admin mutation happens to spawn a fresh worker.
|
||||
///
|
||||
/// Convergence is the whole point of this path, so a rejection is retried on
|
||||
/// the same channel. The worker's exponential backoff caps the cost at one
|
||||
/// reload every `TIER_CONFIG_RELOAD_RETRY_CAP`, and `Terminal` stays reachable
|
||||
/// for transport and gRPC status failures, which is where a genuinely
|
||||
/// unrecoverable peer surfaces.
|
||||
fn tier_config_reload_remote_failure(error_info: Option<String>) -> TierConfigReloadOutcome {
|
||||
let error_info = error_info.unwrap_or_default();
|
||||
if matches!(error_info.as_str(), "errServerNotInitialized" | "ServerNotInitialized") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info))
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(Error::other(error_info))
|
||||
}
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(Error::other(error_info.unwrap_or_default()))
|
||||
}
|
||||
|
||||
fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadOutcome {
|
||||
@@ -1726,6 +1850,14 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else if status.code() == Code::Unknown && status.message().starts_with("Service was not ready:") {
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(status.into())
|
||||
} else if status.code() == Code::Unknown
|
||||
&& is_tier_config_reload_connection_failure(&Error::other(status.message().to_string()))
|
||||
{
|
||||
// tonic reports a connection dropped mid-call as `Unknown` carrying the
|
||||
// transport error text rather than as `Unavailable`, which is what a peer
|
||||
// restarting under an active mutation produces. Reconnect and retry, so
|
||||
// the restart does not permanently retire this peer's reload worker.
|
||||
TierConfigReloadOutcome::TransientReconnect(status.into())
|
||||
} else {
|
||||
TierConfigReloadOutcome::Terminal(status.into())
|
||||
}
|
||||
@@ -1735,9 +1867,13 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use temp_env::async_with_vars;
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[test]
|
||||
@@ -1855,30 +1991,115 @@ mod tests {
|
||||
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
|
||||
let slots = vec![
|
||||
("127.0.0.1:9000".to_string(), None, true),
|
||||
("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false),
|
||||
(
|
||||
"rustfs-1.invalid:9001".to_string(),
|
||||
Some("http://rustfs-1.invalid:9001".to_string()),
|
||||
false,
|
||||
),
|
||||
("rustfs-2.invalid".to_string(), Some("http://rustfs-2.invalid".to_string()), false),
|
||||
("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false),
|
||||
("127.0.0.1:9003".to_string(), None, false),
|
||||
];
|
||||
|
||||
let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots);
|
||||
|
||||
assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away");
|
||||
assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape");
|
||||
assert_eq!(remote.len(), 4, "local node is excluded but remote slots are not compacted away");
|
||||
assert_eq!(all.len(), 5, "all slots preserve the sorted cluster topology shape");
|
||||
assert_eq!(
|
||||
remote_topology_hosts,
|
||||
vec![
|
||||
"127.0.0.1:9001".to_string(),
|
||||
"rustfs-1.invalid:9001".to_string(),
|
||||
"rustfs-2.invalid".to_string(),
|
||||
"127.0.0.1:notaport".to_string(),
|
||||
"127.0.0.1:9003".to_string()
|
||||
]
|
||||
);
|
||||
assert!(remote[0].is_some(), "valid remote peer should get a client");
|
||||
assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot");
|
||||
assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot");
|
||||
let unresolved = remote[0]
|
||||
.as_ref()
|
||||
.expect("temporarily unresolved remote peer should retain a client");
|
||||
assert_eq!(unresolved.host.to_string(), "rustfs-1.invalid:9001");
|
||||
let default_port = remote[1]
|
||||
.as_ref()
|
||||
.expect("temporarily unresolved scheme-default remote peer should retain a client");
|
||||
assert_eq!(default_port.host.to_string(), "rustfs-2.invalid");
|
||||
assert_eq!(default_port.host.port, 80);
|
||||
assert!(!default_port.host.is_port_set);
|
||||
assert!(remote[2].is_none(), "unparseable remote peer should remain observable as a missing slot");
|
||||
assert!(remote[3].is_none(), "missing grid host should remain observable as a missing slot");
|
||||
assert!(all[0].is_none(), "local node is represented by the local server_info row");
|
||||
assert!(all[1].is_some());
|
||||
assert!(all[2].is_none());
|
||||
assert!(all[2].is_some());
|
||||
assert!(all[3].is_none());
|
||||
assert!(all[4].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_host_parser_preserves_names_and_bracketed_ipv6() {
|
||||
let domain = PeerRestClient::parse_topology_host("rustfs-1.invalid", "https://rustfs-1.invalid")
|
||||
.expect("unresolved HTTPS topology host should parse without DNS");
|
||||
assert_eq!(domain.to_string(), "rustfs-1.invalid");
|
||||
assert_eq!(domain.port, 443);
|
||||
assert!(!domain.is_port_set);
|
||||
|
||||
let ipv6 = PeerRestClient::parse_topology_host("[2001:db8::1]:9000", "http://[2001:db8::1]:9000")
|
||||
.expect("bracketed IPv6 topology host should parse without changing its identity");
|
||||
assert_eq!(ipv6.to_string(), "[2001:db8::1]:9000");
|
||||
|
||||
let default_port_ipv6 = PeerRestClient::parse_topology_host("[2001:db8::2]", "http://[2001:db8::2]")
|
||||
.expect("scheme-default IPv6 topology host should parse without DNS");
|
||||
assert_eq!(default_port_ipv6.to_string(), "[2001:db8::2]");
|
||||
assert_eq!(default_port_ipv6.port, 80);
|
||||
assert!(!default_port_ipv6.is_port_set);
|
||||
|
||||
assert!(PeerRestClient::parse_topology_host("peer.invalid:0", "http://peer.invalid:0").is_err());
|
||||
assert!(PeerRestClient::parse_topology_host("peer-a.invalid:9000", "http://peer-b.invalid:9000").is_err());
|
||||
assert!(PeerRestClient::parse_topology_host("peer.invalid:9000", "http://peer.invalid:9000/unexpected").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unresolved_default_port_endpoint_topology_retains_all_peer_clients() {
|
||||
let volumes = (0..4)
|
||||
.map(|index| format!("http://rustfs-{index}.invalid:80/data{index}"))
|
||||
.collect::<Vec<_>>();
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("distributed default-port topology should parse");
|
||||
|
||||
async_with_vars(
|
||||
[
|
||||
(ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("orchestrated")),
|
||||
(ENV_LOCAL_ENDPOINT_HOST, Some("rustfs-0.invalid")),
|
||||
(ENV_KUBERNETES_SERVICE_HOST, None),
|
||||
],
|
||||
async {
|
||||
let (server_pools, setup_type) = EndpointServerPools::create_server_endpoints("0.0.0.0:80", &layout)
|
||||
.await
|
||||
.expect("explicit local identity should avoid peer DNS during endpoint construction");
|
||||
assert_eq!(setup_type, SetupType::DistErasure);
|
||||
|
||||
let (remote, all, remote_topology_hosts) =
|
||||
PeerRestClient::build_clients_from_slots(server_pools.peer_grid_host_slots_sorted());
|
||||
assert_eq!(remote.len(), 3);
|
||||
assert!(
|
||||
remote.iter().all(Option::is_some),
|
||||
"unresolved remote peers must retain reconnectable clients"
|
||||
);
|
||||
assert_eq!(all.len(), 4);
|
||||
assert_eq!(all.iter().filter(|client| client.is_none()).count(), 1);
|
||||
assert_eq!(remote_topology_hosts.len(), 3);
|
||||
assert!(
|
||||
remote_topology_hosts.iter().all(|host| !host.contains(':')),
|
||||
"scheme-default ports must preserve the legacy topology identity"
|
||||
);
|
||||
assert!(
|
||||
remote
|
||||
.iter()
|
||||
.flatten()
|
||||
.all(|client| client.host.port == 80 && !client.host.is_port_set),
|
||||
"scheme-default peers must retain the effective dial port"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2279,9 +2500,12 @@ mod tests {
|
||||
tier_config_reload_status_outcome(tonic::Status::cancelled("request cancelled")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
// A peer that answered and then refused the apply is retried rather than
|
||||
// retired: the channel is healthy, so the rejection reflects remote state
|
||||
// that the next attempt can find healed.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("backend unavailable".to_string())),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(Some("errServerNotInitialized".to_string())),
|
||||
@@ -2305,6 +2529,50 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
/// A tier mutation issued while another node restarts must still converge on
|
||||
/// the nodes that stayed up. Those peers answer the reload RPC and reject the
|
||||
/// apply, because reloading reads the tier mutation intents and takes the
|
||||
/// distributed tier-config lock while the lock quorum is still disturbed.
|
||||
/// Classifying those rejections as terminal retired the reload worker on its
|
||||
/// first attempt and pinned the peer to the previous configuration, so it
|
||||
/// served `TierNotFound` for an already-committed tier until an unrelated
|
||||
/// second admin mutation spawned a new worker.
|
||||
#[test]
|
||||
fn tier_config_reload_retries_peers_that_reject_the_apply_mid_restart() {
|
||||
for error_info in [
|
||||
"Lock acquisition timeout for resource '.rustfs.sys/config/tier-config.bin.lock' after 5s",
|
||||
"Resource '.rustfs.sys/config/tier-config.bin.lock' is already locked by node-3",
|
||||
"Internal error: release RPC failed: transport error",
|
||||
"save_config_with_opts: err: PreconditionFailed",
|
||||
"erasure read quorum",
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
tier_config_reload_remote_failure(Some(error_info.to_string())),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
),
|
||||
"a peer that rejected the apply must stay retryable so it converges: {error_info}"
|
||||
);
|
||||
}
|
||||
|
||||
// An absent error message is still a rejection, not a reason to stop.
|
||||
assert!(matches!(
|
||||
tier_config_reload_remote_failure(None),
|
||||
TierConfigReloadOutcome::TransientRetrySameChannel(_)
|
||||
));
|
||||
|
||||
// tonic surfaces a connection dropped mid-call as `Unknown`, not `Unavailable`.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("transport error")),
|
||||
TierConfigReloadOutcome::TransientReconnect(_)
|
||||
));
|
||||
// An `Unknown` that is not transport-shaped stays terminal.
|
||||
assert!(matches!(
|
||||
tier_config_reload_status_outcome(tonic::Status::unknown("peer response unknown")),
|
||||
TierConfigReloadOutcome::Terminal(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_config_reload_single_attempt_clears_offline_gate_without_redial() {
|
||||
let client = test_peer_client();
|
||||
@@ -2381,6 +2649,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
|
||||
epoch
|
||||
);
|
||||
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
|
||||
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
@@ -2605,6 +2889,31 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn dropped_peer_client_releases_and_stops_its_recovery_monitor() {
|
||||
let client = test_peer_client();
|
||||
client.offline.store(true, Ordering::Release);
|
||||
client.recovery_running.store(true, Ordering::Release);
|
||||
let offline = Arc::downgrade(&client.offline);
|
||||
let recovery_running = Arc::downgrade(&client.recovery_running);
|
||||
let handle = PeerRestClient::spawn_recovery_monitor(client.grid_host.clone(), offline.clone(), recovery_running.clone());
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
drop(client);
|
||||
|
||||
assert!(offline.upgrade().is_none(), "detached recovery must not retain offline state");
|
||||
assert!(
|
||||
recovery_running.upgrade().is_none(),
|
||||
"detached recovery must not retain its running state"
|
||||
);
|
||||
handle.await.expect("recovery monitor should not panic");
|
||||
assert_eq!(
|
||||
tokio::time::Instant::now(),
|
||||
started,
|
||||
"recovery monitor should stop before advancing to its first delayed probe"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() {
|
||||
// Regression: application error text containing "unavailable" (a
|
||||
@@ -2652,6 +2961,11 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` callsite is shared with the production
|
||||
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
|
||||
// subscriber-less threads; without this the span can be cached as
|
||||
// `Interest::never()` and silently degrade to `Span::none()`.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let client = test_peer_client();
|
||||
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
|
||||
|
||||
@@ -14,8 +14,10 @@
|
||||
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||
@@ -39,16 +41,84 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::{
|
||||
Mutex as StdMutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::{net::TcpStream, sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
type Client = Arc<Box<dyn PeerS3Client>>;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct DeleteBucketEmptyScanBarrier {
|
||||
arrived: AtomicBool,
|
||||
arrived_notify: Notify,
|
||||
released: AtomicBool,
|
||||
release_notify: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteBucketEmptyScanBarrier {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
loop {
|
||||
let notified = self.arrived_notify.notified();
|
||||
if self.arrived.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.released.store(true, Ordering::Release);
|
||||
self.release_notify.notify_waiters();
|
||||
}
|
||||
|
||||
async fn pause(&self) {
|
||||
self.arrived.store(true, Ordering::Release);
|
||||
self.arrived_notify.notify_waiters();
|
||||
loop {
|
||||
let notified = self.release_notify.notified();
|
||||
if self.released.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
|
||||
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
|
||||
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
|
||||
barrier
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_after_delete_bucket_empty_scan() {
|
||||
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned")
|
||||
.take();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.pause().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerBucketListing {
|
||||
pub buckets: Vec<BucketInfo>,
|
||||
@@ -89,6 +159,22 @@ fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option<Error>]) -> Option<Erro
|
||||
reduce_write_quorum_errs(per_pool_errs, BUCKET_OP_IGNORED_ERRS, pool_write_quorum(per_pool_errs.len()))
|
||||
}
|
||||
|
||||
fn resolve_heal_bucket_mode(opts: &mut HealOpts, pool_errs: &[Option<Error>]) -> Result<()> {
|
||||
if opts.recreate {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(err) = pool_errs
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|err| **err != Error::DiskNotFound && **err != Error::VolumeNotFound)
|
||||
{
|
||||
return Err(err.clone());
|
||||
}
|
||||
opts.remove = is_all_buckets_not_found(pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait PeerS3Client: Debug + Sync + Send + 'static {
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
|
||||
@@ -159,10 +245,7 @@ impl S3PeerSys {
|
||||
pool_errs.push(reduce_pool_write_quorum_errs(&per_pool_errs));
|
||||
}
|
||||
|
||||
if !opts.recreate {
|
||||
opts.remove = is_all_buckets_not_found(&pool_errs);
|
||||
opts.recreate = !opts.remove;
|
||||
}
|
||||
resolve_heal_bucket_mode(&mut opts, &pool_errs)?;
|
||||
|
||||
let mut futures = Vec::new();
|
||||
let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
|
||||
@@ -636,24 +719,22 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let force = if opts.force_if_empty && !opts.force {
|
||||
if opts.force_if_empty && !opts.force {
|
||||
for disk in local_disks.iter() {
|
||||
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
|
||||
return Err(Error::VolumeNotEmpty);
|
||||
}
|
||||
}
|
||||
true
|
||||
} else {
|
||||
opts.force
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_after_delete_bucket_empty_scan().await;
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(local_disks.len());
|
||||
|
||||
for disk in local_disks.iter() {
|
||||
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
|
||||
// the recreate loop below turns into BucketNotEmpty; only an explicit
|
||||
// force delete removes recursively (backlog#799 B1).
|
||||
futures.push(disk.delete_volume(bucket, force));
|
||||
// `force_if_empty` is validation-only. Passing it as force would let
|
||||
// a PutObject committed after the scan be removed recursively.
|
||||
futures.push(disk.delete_volume(bucket, opts.force));
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
@@ -716,6 +797,15 @@ pub struct RemotePeerS3Client {
|
||||
}
|
||||
|
||||
impl RemotePeerS3Client {
|
||||
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
|
||||
let mut remote_opts = opts.clone();
|
||||
// Older peers promote `force_if_empty` to recursive force after their
|
||||
// metadata scan. Keep this coordinator-only hint off the wire so a
|
||||
// mixed-version delete fails closed on non-empty directory remnants.
|
||||
remote_opts.force_if_empty = false;
|
||||
serde_json::to_string(&remote_opts).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn recovery_monitor_span(addr: &str) -> tracing::Span {
|
||||
tracing::info_span!(
|
||||
"recovery-monitor",
|
||||
@@ -742,7 +832,7 @@ impl RemotePeerS3Client {
|
||||
client
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
||||
@@ -917,10 +1007,11 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options: String = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(HealBucketRequest {
|
||||
let mut request = Request::new(HealBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.heal_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -973,10 +1064,11 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
|| async {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
let request = Request::new(MakeBucketRequest {
|
||||
let mut request = Request::new(MakeBucketRequest {
|
||||
name: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.make_bucket(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
@@ -1024,13 +1116,14 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let options = Self::encode_delete_bucket_options(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
let request = Request::new(DeleteBucketRequest {
|
||||
let mut request = Request::new(DeleteBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let response = client.delete_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return if let Some(err) = response.error {
|
||||
@@ -1397,6 +1490,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
no_lock: true,
|
||||
no_recreate: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote delete options should serialize");
|
||||
let legacy_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
|
||||
|
||||
assert!(legacy_opts.no_lock);
|
||||
assert!(legacy_opts.no_recreate);
|
||||
assert!(!legacy_opts.force);
|
||||
assert!(!legacy_opts.force_if_empty);
|
||||
|
||||
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
|
||||
true
|
||||
} else {
|
||||
legacy_opts.force
|
||||
};
|
||||
assert!(
|
||||
!legacy_recursive_force,
|
||||
"legacy peer must not upgrade empty-only delete to recursive force"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_preserve_explicit_force() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
force: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote force-delete options should serialize");
|
||||
let remote_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
|
||||
|
||||
assert!(remote_opts.force);
|
||||
assert!(!remote_opts.force_if_empty);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
|
||||
let client = test_remote_peer("http://peer-network-error:9000");
|
||||
@@ -1554,6 +1690,54 @@ mod tests {
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
|
||||
let disks = init_test_local_disks_for_pools(
|
||||
&temp_dir,
|
||||
&[(0, 1), (1, 1)],
|
||||
"local-peer-force-if-empty-preserves-unclassified-file",
|
||||
)
|
||||
.await;
|
||||
let bucket = "empty-only-delete-bucket";
|
||||
let marker = "object/commit-marker";
|
||||
let data = bytes::Bytes::from_static(b"committed object data");
|
||||
|
||||
disks[1]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should be created in the selected pool");
|
||||
disks[1]
|
||||
.write_all(bucket, marker, data.clone())
|
||||
.await
|
||||
.expect("unclassified committed file should be written");
|
||||
|
||||
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("empty-only delete must not recursively remove an unclassified file");
|
||||
|
||||
assert_eq!(err, Error::VolumeNotEmpty);
|
||||
assert_eq!(
|
||||
disks[1]
|
||||
.read_all(bucket, marker)
|
||||
.await
|
||||
.expect("unclassified committed file should be preserved"),
|
||||
data
|
||||
);
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
|
||||
@@ -1618,6 +1802,30 @@ mod tests {
|
||||
assert_eq!(err, Error::VolumeExists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_fails_closed_on_incomplete_topology() {
|
||||
let mut opts = HealOpts::default();
|
||||
assert_eq!(
|
||||
resolve_heal_bucket_mode(&mut opts, &[Some(Error::ErasureWriteQuorum)]),
|
||||
Err(Error::ErasureWriteQuorum)
|
||||
);
|
||||
assert!(!opts.recreate);
|
||||
assert!(!opts.remove);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_bucket_mode_distinguishes_deleted_and_partial_buckets() {
|
||||
let mut deleted = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut deleted, &[Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(deleted.remove);
|
||||
assert!(!deleted.recreate);
|
||||
|
||||
let mut partial = HealOpts::default();
|
||||
resolve_heal_bucket_mode(&mut partial, &[None, Some(Error::VolumeNotFound)]).unwrap();
|
||||
assert!(!partial.remove);
|
||||
assert!(partial.recreate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_bucket_reduces_quorum_by_pool_participants() {
|
||||
let peer_sys = S3PeerSys {
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
@@ -25,7 +25,7 @@ use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -50,8 +50,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -70,7 +71,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -100,6 +101,18 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_DISK: &str = "remote_disk";
|
||||
const EVENT_REMOTE_DISK_HEALTH: &str = "remote_disk_health";
|
||||
const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
|
||||
|
||||
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.protocol_version != SNAPSHOT_LEASE_PROTOCOL_VERSION {
|
||||
return Err(Error::other("remote snapshot lease protocol is incompatible"));
|
||||
}
|
||||
SnapshotLeaseToken::from_slice(&response.token)
|
||||
}
|
||||
|
||||
/// Bind a mutating disk RPC to its canonical body: the digest lands in the request metadata, and
|
||||
/// the signing interceptor folds it (plus a replay-protected nonce) into the v2 signature scope
|
||||
@@ -1070,7 +1083,7 @@ impl RemoteDisk {
|
||||
internode_offline_bypass_reason(&self.addr).map(Error::other)
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1083,7 +1096,7 @@ impl RemoteDisk {
|
||||
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
|
||||
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
|
||||
/// is disabled.
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1784,6 +1797,81 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn acquire_snapshot_lease(&self, volume: &str, path: &str) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
|
||||
let response = client.acquire_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseRenewRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
ttl_ms: u64::try_from(REMOTE_SNAPSHOT_LEASE_TTL.as_millis())
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_renew_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "renew_snapshot_lease")?;
|
||||
let response = client.renew_snapshot_lease(request).await?.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SnapshotLeaseReleaseRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
|
||||
let response = client.release_snapshot_lease(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
trace!(
|
||||
@@ -2917,6 +3005,7 @@ mod tests {
|
||||
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::io::{self as std_io, Write};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
|
||||
@@ -2930,6 +3019,48 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
// `#[serial(internode_metrics)]` marks every test that observes
|
||||
// `global_internode_metrics()`. Those counters are a process-wide singleton:
|
||||
// some of these tests snapshot a counter, run one decode, and assert on the
|
||||
// delta, while others deliberately record decode errors or call
|
||||
// `reset_internode_metrics_for_test()`. Run concurrently in one process they
|
||||
// corrupt each other's deltas — a sibling's error bumps the "no decode error"
|
||||
// assertion off zero, and a sibling's reset can drive an `after > before`
|
||||
// assertion backwards.
|
||||
//
|
||||
// The marker only takes effect under the `cargo test` fallback; nextest
|
||||
// already isolates each test in its own process, so every test there gets its
|
||||
// own copy of the counters (see `docs/testing/README.md`). Any new test that
|
||||
// reads or mutates the global internode metrics belongs in this group.
|
||||
|
||||
#[test]
|
||||
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
|
||||
let token = SnapshotLeaseToken::new();
|
||||
let response = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(snapshot_lease_token_from_response(response).unwrap(), token);
|
||||
|
||||
let incompatible = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION + 1,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(incompatible).is_err());
|
||||
|
||||
let malformed = SnapshotLeaseResponse {
|
||||
success: true,
|
||||
token: Bytes::from_static(b"not-a-uuid"),
|
||||
protocol_version: SNAPSHOT_LEASE_PROTOCOL_VERSION,
|
||||
error: None,
|
||||
};
|
||||
assert!(snapshot_lease_token_from_response(malformed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_volumes_decode_rejects_a_malformed_entry() {
|
||||
let valid = serde_json::to_string(&VolumeInfo {
|
||||
@@ -3190,6 +3321,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
|
||||
@@ -3212,6 +3344,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_falls_back_to_json_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let json_resp = sample_read_multiple_resp("json", b"fallback");
|
||||
@@ -3233,6 +3366,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn rename_data_response_accepts_legacy_json_without_decode_error() {
|
||||
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
|
||||
let response = RenameDataResp {
|
||||
@@ -3384,6 +3518,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3409,6 +3544,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3449,6 +3585,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
|
||||
@@ -3469,6 +3606,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_rejects_invalid_success_metadata() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
|
||||
@@ -3488,6 +3626,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -3514,6 +3653,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -4303,6 +4443,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
|
||||
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
|
||||
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
@@ -4341,6 +4482,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
|
||||
// A transient reset-by-peer on a shard read during the read-after-write window must be
|
||||
// absorbed by one re-dial rather than eroding read quorum (issue #2761).
|
||||
@@ -4672,9 +4814,11 @@ mod tests {
|
||||
async fn test_remote_disk_endpoints_with_different_schemes() {
|
||||
let test_cases = vec![
|
||||
("http://server:9000", "server:9000"),
|
||||
("https://secure-server:443", "secure-server"), // Default HTTPS port is omitted
|
||||
("http://plain-server:80", "plain-server"),
|
||||
("http://plain-server", "plain-server"),
|
||||
("https://secure-server:443", "secure-server"),
|
||||
("http://192.168.1.100:8080", "192.168.1.100:8080"),
|
||||
("https://secure-server", "secure-server"), // No port specified
|
||||
("https://secure-server", "secure-server"),
|
||||
];
|
||||
|
||||
for (url_str, expected_hostname) in test_cases {
|
||||
@@ -5190,6 +5334,11 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
|
||||
@@ -5244,6 +5393,11 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let addr = "http://127.0.0.1:59997".to_string();
|
||||
let endpoint = Endpoint {
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use rustfs_lock::{
|
||||
@@ -28,7 +31,6 @@ use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Remote lock client implementation
|
||||
@@ -77,7 +79,7 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
|
||||
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
|
||||
// change quorum; the self-healing re-probe keeps the peer recoverable.
|
||||
@@ -313,10 +315,11 @@ impl LockClient for RemoteClient {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
|
||||
Ok(resp) => resp.into_inner(),
|
||||
@@ -347,7 +350,7 @@ impl LockClient for RemoteClient {
|
||||
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -355,6 +358,7 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
|
||||
@@ -395,7 +399,8 @@ impl LockClient for RemoteClient {
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("release", &resource_summary, client.un_lock(req))
|
||||
.await?
|
||||
@@ -414,7 +419,7 @@ impl LockClient for RemoteClient {
|
||||
let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = Self::summarize_resources(&unlock_requests);
|
||||
let req = Request::new(BatchGenerallyLockRequest {
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: unlock_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
@@ -422,6 +427,7 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = self
|
||||
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
|
||||
@@ -440,10 +446,11 @@ impl LockClient for RemoteClient {
|
||||
let refresh_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = refresh_request.resource.to_string();
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&refresh_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("refresh", &resource_summary, client.refresh(req))
|
||||
.await?
|
||||
@@ -459,10 +466,11 @@ impl LockClient for RemoteClient {
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = force_request.resource.to_string();
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&force_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
|
||||
.await?
|
||||
@@ -483,10 +491,11 @@ impl LockClient for RemoteClient {
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
|
||||
@@ -497,10 +506,11 @@ impl LockClient for RemoteClient {
|
||||
if resp.success {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let release_req = Request::new(GenerallyLockRequest {
|
||||
let mut release_req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut release_req)?;
|
||||
let _ = self
|
||||
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
|
||||
.await;
|
||||
|
||||
+199
-20
@@ -37,7 +37,9 @@ use crate::{
|
||||
runtime::instance::{InstanceContext, bootstrap_ctx},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::{PreparedGetObjectMetadata, SetDisks},
|
||||
store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
|
||||
store::init_format::{
|
||||
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
|
||||
},
|
||||
};
|
||||
use futures::{
|
||||
future::join_all,
|
||||
@@ -947,7 +949,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let (disks, _) = init_storage_disks_with_errors(
|
||||
let (disks, init_errs) = init_storage_disks_with_errors(
|
||||
&self.endpoints.endpoints,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -955,15 +957,36 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (formats, errs) = load_format_erasure_all(&disks, true).await;
|
||||
let (formats, mut errs) = load_format_erasure_all(&disks, true).await;
|
||||
for (err, init_err) in errs.iter_mut().zip(init_errs) {
|
||||
if init_err.is_some() {
|
||||
*err = init_err;
|
||||
}
|
||||
}
|
||||
if errs.iter().any(|err| {
|
||||
matches!(
|
||||
err,
|
||||
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
|
||||
)
|
||||
}) {
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
|
||||
info!("failed to check formats erasure values: {}", err);
|
||||
return Ok((HealResultItem::default(), Some(err)));
|
||||
}
|
||||
let ref_format = match get_format_erasure_in_quorum(&formats) {
|
||||
Ok(format) => format,
|
||||
let (ref_format, quorum_members) = match select_format_erasure_in_quorum(&formats, 0) {
|
||||
Ok((format, members)) if format.shared_identity() == self.format.shared_identity() => (format, members),
|
||||
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
|
||||
Err(err) => return Ok((HealResultItem::default(), Some(err))),
|
||||
};
|
||||
if formats
|
||||
.iter()
|
||||
.zip(quorum_members)
|
||||
.any(|(format, member)| format.is_some() && !member)
|
||||
{
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
let mut res = HealResultItem {
|
||||
heal_item_type: HealItemType::Metadata.to_string(),
|
||||
detail: "disk-format".to_string(),
|
||||
@@ -985,11 +1008,6 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
return Ok((res, Some(StorageError::NoHealRequired)));
|
||||
}
|
||||
|
||||
// if !self.format.eq(&ref_format) {
|
||||
// info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format);
|
||||
// return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
|
||||
// }
|
||||
|
||||
let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);
|
||||
if !dry_run {
|
||||
let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count];
|
||||
@@ -1298,7 +1316,7 @@ mod tests {
|
||||
assert_eq!(result, (Some(3), Some(1), Some(0)));
|
||||
}
|
||||
|
||||
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
async fn two_set_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
let format = FormatV3::new(2, 2);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
@@ -1339,8 +1357,8 @@ mod tests {
|
||||
Arc::new(RwLock::new(disks)),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
set_index,
|
||||
0,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
|
||||
@@ -1373,11 +1391,114 @@ mod tests {
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_format_heal_accepts_quorum_from_a_nonzero_set() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
|
||||
let (result, err) = sets.disk_set[1]
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("the second erasure set should load its own format quorum");
|
||||
|
||||
assert!(matches!(err, Some(StorageError::NoHealRequired)), "unexpected heal result: {err:?}");
|
||||
assert_eq!(result.disk_count, 2);
|
||||
assert_eq!(result.set_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_rejects_foreign_majorities_at_set_and_pool_scopes() {
|
||||
let (_temp_dirs, _canonical_format, sets) = setup_heal_format_sets(2, true).await;
|
||||
let set_disks = set_level_heal_view(&sets).await;
|
||||
|
||||
let (_, set_err) = set_disks
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("set format heal should report a typed mismatch");
|
||||
assert!(
|
||||
matches!(set_err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign set majority must not replace the cached format: {set_err:?}"
|
||||
);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign pool majority must not replace the cached format: {pool_err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_format_heal_rejects_a_wrong_slot_minority() {
|
||||
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
let mut poisoned_format = canonical_format.clone();
|
||||
poisoned_format.erasure.this = canonical_format.erasure.sets[0][0];
|
||||
replace_heal_test_format(&sets, 2, &poisoned_format).await;
|
||||
let probe_err = new_disk(
|
||||
&sets.endpoints.endpoints.as_ref()[2],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a wrong-slot local format must fail disk initialization");
|
||||
assert_eq!(probe_err, DiskError::InconsistentDisk);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed slot mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"a wrong-slot minority must not be reported as no-heal-required: {pool_err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_heal_test_format(&sets, 2).await,
|
||||
poisoned_format,
|
||||
"format heal must not overwrite a wrong-slot disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_rejects_a_foreign_minority_at_set_and_pool_scopes() {
|
||||
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
let mut poisoned_format = canonical_format.clone();
|
||||
poisoned_format.id = Uuid::new_v4();
|
||||
poisoned_format.erasure.this = poisoned_format.erasure.sets[0][2];
|
||||
replace_heal_test_format(&sets, 2, &poisoned_format).await;
|
||||
let set_disks = set_level_heal_view(&sets).await;
|
||||
|
||||
let (_, set_err) = set_disks
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("set format heal should report a typed identity mismatch");
|
||||
assert!(
|
||||
matches!(set_err, Some(StorageError::CorruptedFormat)),
|
||||
"a foreign minority must not be reported as no-heal-required: {set_err:?}"
|
||||
);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed identity mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"a foreign minority must not be reported as no-heal-required: {pool_err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_heal_test_format(&sets, 2).await,
|
||||
poisoned_format,
|
||||
"format heal must not overwrite a foreign disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
|
||||
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
@@ -1616,11 +1737,15 @@ mod tests {
|
||||
// formatting the first `num_formatted` of them against a shared reference
|
||||
// format and leaving the rest unformatted. Returns the live TempDir handles
|
||||
// (must be kept alive), the reference format, and the assembled `Sets`.
|
||||
// `disk_set` is intentionally empty: these tests only drive `heal_format`
|
||||
// with `dry_run == true`, which never touches `disk_set`.
|
||||
async fn setup_heal_format_sets(num_formatted: usize) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
|
||||
// `disk_set` is intentionally empty: these tests only exercise paths that
|
||||
// return before pool-level healing delegates into a set.
|
||||
async fn setup_heal_format_sets(num_formatted: usize, foreign_identity: bool) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
|
||||
const SET_DRIVE_COUNT: usize = 3;
|
||||
let ref_format = FormatV3::new(1, SET_DRIVE_COUNT);
|
||||
let mut stored_format = ref_format.clone();
|
||||
if foreign_identity {
|
||||
stored_format.id = Uuid::new_v4();
|
||||
}
|
||||
|
||||
let mut dirs = Vec::with_capacity(SET_DRIVE_COUNT);
|
||||
let mut endpoints = Vec::with_capacity(SET_DRIVE_COUNT);
|
||||
@@ -1645,8 +1770,8 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
let mut disk_format = ref_format.clone();
|
||||
disk_format.erasure.this = ref_format.erasure.sets[0][i];
|
||||
let mut disk_format = stored_format.clone();
|
||||
disk_format.erasure.this = stored_format.erasure.sets[0][i];
|
||||
save_format_file(&Some(disk), &Some(disk_format))
|
||||
.await
|
||||
.expect("format should be saved");
|
||||
@@ -1677,6 +1802,60 @@ mod tests {
|
||||
(dirs, ref_format, sets)
|
||||
}
|
||||
|
||||
async fn set_level_heal_view(sets: &Sets) -> Arc<SetDisks> {
|
||||
let endpoints = sets.endpoints.endpoints.as_ref().clone();
|
||||
let mut disks = Vec::with_capacity(endpoints.len());
|
||||
for endpoint in &endpoints {
|
||||
disks.push(Some(
|
||||
new_disk(
|
||||
endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fresh set-level disk handle should open"),
|
||||
));
|
||||
}
|
||||
|
||||
SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
endpoints.len(),
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
sets.format.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn replace_heal_test_format(sets: &Sets, disk_index: usize, format: &FormatV3) {
|
||||
let disk = new_disk(
|
||||
&sets.endpoints.endpoints.as_ref()[disk_index],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal test disk should open");
|
||||
save_format_file(&Some(disk.clone()), &Some(format.clone()))
|
||||
.await
|
||||
.expect("poisoned test format should be written");
|
||||
}
|
||||
|
||||
async fn read_heal_test_format(sets: &Sets, disk_index: usize) -> FormatV3 {
|
||||
let path = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[disk_index].get_file_path())
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(crate::disk::FORMAT_CONFIG_FILE);
|
||||
let data = tokio::fs::read(path).await.expect("test format should be readable");
|
||||
FormatV3::try_from(data.as_slice()).expect("test format should parse")
|
||||
}
|
||||
|
||||
// Regression for #956 (NoHealRequired path): with every disk already
|
||||
// formatted, `heal_format` reports exactly one drive record per disk
|
||||
// (N = set_count * set_drive_count), each carrying a real endpoint. Before
|
||||
@@ -1685,7 +1864,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_format_no_heal_required_reports_one_record_per_disk() {
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3).await;
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
|
||||
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
|
||||
// All disks formatted -> NoHealRequired early return, still returns `res`.
|
||||
@@ -1715,7 +1894,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn heal_format_heal_path_reports_one_record_per_disk_aligned() {
|
||||
// Disks 0 and 1 formatted (quorum), disk 2 unformatted.
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2).await;
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2, false).await;
|
||||
|
||||
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
|
||||
// Unformatted disk present -> heal path, not NoHealRequired.
|
||||
|
||||
@@ -1365,6 +1365,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.renew_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.delete_data_dir(volume, path, opts).await },
|
||||
|
||||
+1229
-29
File diff suppressed because it is too large
Load Diff
@@ -79,6 +79,18 @@ impl SnapshotLeaseToken {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::new_v4())
|
||||
}
|
||||
|
||||
pub fn from_slice(bytes: &[u8]) -> Result<Self> {
|
||||
let uuid = Uuid::from_slice(bytes).map_err(|_| Error::other("invalid snapshot lease token"))?;
|
||||
if uuid.is_nil() {
|
||||
return Err(Error::other("invalid snapshot lease token"));
|
||||
}
|
||||
Ok(Self(uuid))
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||
self.0.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SnapshotLeaseToken {
|
||||
@@ -284,6 +296,13 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.renew_snapshot_lease(volume, path, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.delete_data_dir(volume, path, opts).await,
|
||||
@@ -694,6 +713,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
async fn release_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<()> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn renew_snapshot_lease(&self, _volume: &str, _path: &str, _token: SnapshotLeaseToken) -> Result<SnapshotLeaseToken> {
|
||||
Err(Error::other("snapshot leases are not supported by this disk"))
|
||||
}
|
||||
async fn delete_data_dir(&self, volume: &str, path: &str, opts: DeleteOptions) -> Result<DataDirDeleteStatus> {
|
||||
self.delete(volume, path, opts).await?;
|
||||
Ok(DataDirDeleteStatus::Deleted)
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::{
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
|
||||
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
|
||||
use tracing::warn;
|
||||
|
||||
/// Check path length according to OS limits.
|
||||
@@ -123,6 +123,8 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
|
||||
|
||||
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
|
||||
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
|
||||
let cpu_scaled = cpu_count
|
||||
@@ -157,6 +159,24 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
|
||||
limiter
|
||||
}
|
||||
|
||||
/// Serialize a bucket's local metadata commits with physical bucket removal.
|
||||
///
|
||||
/// The key includes the canonical disk root, so independently reconnected
|
||||
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
|
||||
/// disconnected disks do not keep the registry alive.
|
||||
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
|
||||
let key = root.join(volume);
|
||||
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let lock = Arc::new(RwLock::new(()));
|
||||
locks.insert(key, Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
/// Always acquire the per-disk permit before the process-wide permit. Keeping
|
||||
/// this order uniform prevents one slow disk from reserving global capacity
|
||||
/// while it waits for its own concurrency slot.
|
||||
@@ -572,15 +592,14 @@ async fn reliable_rename_inner(
|
||||
base_dir: impl AsRef<Path>,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
if let Some(parent) = dst_file_path.as_ref().parent()
|
||||
&& !file_exists(parent)
|
||||
{
|
||||
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
|
||||
}
|
||||
let parent_guard = match dst_file_path.as_ref().parent() {
|
||||
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut i = 0;
|
||||
loop {
|
||||
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
|
||||
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
|
||||
if should_retry_rename(&e, i) {
|
||||
i += 1;
|
||||
continue;
|
||||
@@ -597,6 +616,159 @@ async fn reliable_rename_inner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
use rustix::fs::{Mode, OFlags, open, renameat};
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
return super::fs::rename_std(src_file_path, dst_file_path);
|
||||
};
|
||||
let src_parent = src_file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
|
||||
let src_name = src_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
|
||||
let dst_name = dst_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
|
||||
let src_parent = open(
|
||||
src_parent,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let dst_parent = parent_guard
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
|
||||
|
||||
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
super::fs::rename_std(src_file_path, dst_file_path)
|
||||
}
|
||||
|
||||
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let dir_path = dir_path.to_path_buf();
|
||||
let base_dir = base_dir.to_path_buf();
|
||||
|
||||
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = ();
|
||||
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
|
||||
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
|
||||
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
|
||||
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
|
||||
const FILE_SHARE_READ: u32 = 0x1;
|
||||
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)?;
|
||||
let handle = winapi_util::Handle::from_file(file);
|
||||
let info = winapi_util::file::information(&handle)?;
|
||||
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|
||||
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
|
||||
{
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
}
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let relative = dir_path
|
||||
.strip_prefix(base_dir)
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
|
||||
for component in relative.components() {
|
||||
if !matches!(component, Component::Normal(_) | Component::CurDir) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"rename destination contains an invalid path component",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
|
||||
use rustix::io::Errno;
|
||||
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
|
||||
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
let parent = parents
|
||||
.last()
|
||||
.expect("base directory guard should contain the base directory");
|
||||
match mkdirat(parent, component, mode) {
|
||||
Ok(()) => {}
|
||||
Err(Errno::EXIST) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
|
||||
}
|
||||
|
||||
Ok(parents)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut handles = vec![lock_windows_directory(base_dir)?];
|
||||
let mut current = base_dir.to_path_buf();
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
current.push(component);
|
||||
match std::fs::create_dir(¤t) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
handles.push(lock_windows_directory(¤t)?);
|
||||
}
|
||||
|
||||
Ok(handles)
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = relative;
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"safe recursive directory creation is unavailable on this platform",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
|
||||
warn!(
|
||||
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
|
||||
@@ -727,6 +899,20 @@ mod tests {
|
||||
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
|
||||
|
||||
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
|
||||
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
|
||||
|
||||
let _write_guard = first.write().await;
|
||||
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -771,9 +957,26 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
|
||||
/// the pin that keeps tracing's process-global callsite-interest cache from
|
||||
/// being decided by some other test's thread.
|
||||
struct WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::DefaultGuard,
|
||||
_callsite_pin: tracing::Dispatch,
|
||||
}
|
||||
|
||||
/// Capture WARN-level output on the current thread; tokio tests here run on
|
||||
/// the current-thread runtime, so the guard covers the whole test body.
|
||||
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
|
||||
///
|
||||
/// The callsite pin matters because `warn_reliable_rename_failure` is a
|
||||
/// single production callsite shared with tests that call `rename_all`
|
||||
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
|
||||
/// is one. Whichever thread reaches it first fixes its `Interest`
|
||||
/// process-wide, so without the pin that sibling can cache
|
||||
/// `Interest::never()` and the WARN never fires here at all, leaving the
|
||||
/// "must keep the WARN" assertions staring at empty output. See
|
||||
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
|
||||
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
@@ -781,7 +984,10 @@ mod tests {
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let guard = tracing::subscriber::set_default(subscriber);
|
||||
let guard = WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::set_default(subscriber),
|
||||
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
|
||||
};
|
||||
(logs, guard)
|
||||
}
|
||||
|
||||
@@ -968,6 +1174,112 @@ mod tests {
|
||||
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_all_does_not_recreate_missing_base_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
std::fs::remove_dir(&base).expect("delete destination base before commit");
|
||||
|
||||
let err = rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must not recreate a deleted destination base");
|
||||
|
||||
assert!(matches!(err, DiskError::FileNotFound));
|
||||
assert!(src.exists(), "failed commit must preserve the staged source");
|
||||
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
|
||||
std::fs::remove_dir(&base).expect("remove destination base before replacement");
|
||||
symlink(&outside, &base).expect("replace destination base with a symlink");
|
||||
|
||||
rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must reject an existing destination parent below a replaced base");
|
||||
|
||||
assert!(src.exists(), "rejected rename must preserve the staged source");
|
||||
assert!(
|
||||
!outside.join("object/xl.meta").exists(),
|
||||
"rename must not publish through the replacement symlink"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let parent = base.join("object").join("nested");
|
||||
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
|
||||
|
||||
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
|
||||
.expect_err("the locked base must not be replaceable before commit");
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect_err("a locked intermediate directory must not be replaceable before commit");
|
||||
|
||||
drop(guard);
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect("replacement should succeed after the commit guard is released");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlinked_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
symlink(&outside, &base).expect("create symlinked base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("object"), &base)
|
||||
.await
|
||||
.expect_err("symlinked base must be rejected");
|
||||
|
||||
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlink_below_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
symlink(&outside, base.join("linked")).expect("create symlink below base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
|
||||
.await
|
||||
.expect_err("symlink below base must be rejected");
|
||||
|
||||
assert!(
|
||||
!outside.join("object").exists(),
|
||||
"parent creation must not follow a symlink outside the base"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fsync_dir_succeeds_on_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
|
||||
@@ -203,7 +203,7 @@ fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args:
|
||||
for args in set_args.iter() {
|
||||
for arg in args {
|
||||
if unique_args.contains(arg) {
|
||||
return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
|
||||
return Err(Error::other("input arguments contain a duplicate endpoint after ellipsis expansion"));
|
||||
}
|
||||
unique_args.insert(arg);
|
||||
}
|
||||
@@ -924,4 +924,15 @@ mod test {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_errors_do_not_echo_url_credentials() {
|
||||
for volumes in [
|
||||
vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"],
|
||||
vec!["http://:ellipsis...secret@server/path"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).unwrap_err();
|
||||
assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ impl TryFrom<&str> for Endpoint {
|
||||
// - All field should be empty except Host and Path.
|
||||
if !((url.scheme() == "http" || url.scheme() == "https")
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none()
|
||||
&& url.fragment().is_none()
|
||||
&& url.query().is_none())
|
||||
{
|
||||
@@ -366,6 +367,12 @@ mod test {
|
||||
expected_type: None,
|
||||
expected_err: Some(Error::other("invalid URL endpoint format")),
|
||||
},
|
||||
TestCase {
|
||||
arg: "http://:topsecret@server/path",
|
||||
expected_endpoint: None,
|
||||
expected_type: None,
|
||||
expected_err: Some(Error::other("invalid URL endpoint format")),
|
||||
},
|
||||
TestCase {
|
||||
arg: "http://:/path",
|
||||
expected_endpoint: None,
|
||||
@@ -505,8 +512,18 @@ mod test {
|
||||
let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
|
||||
assert_eq!(endpoint.host_port(), "example.com:9000");
|
||||
|
||||
let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
|
||||
assert_eq!(endpoint_no_port.host_port(), "example.com");
|
||||
for endpoint in [
|
||||
Endpoint::try_from("http://example.com/path").unwrap(),
|
||||
Endpoint::try_from("http://example.com:80/path").unwrap(),
|
||||
] {
|
||||
assert_eq!(endpoint.host_port(), "example.com");
|
||||
}
|
||||
for endpoint in [
|
||||
Endpoint::try_from("https://example.com/path").unwrap(),
|
||||
Endpoint::try_from("https://example.com:443/path").unwrap(),
|
||||
] {
|
||||
assert_eq!(endpoint.host_port(), "example.com");
|
||||
}
|
||||
|
||||
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
|
||||
assert_eq!(file_endpoint.host_port(), "");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Error as JsonError;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum FormatMetaVersion {
|
||||
#[serde(rename = "1")]
|
||||
V1,
|
||||
@@ -27,7 +27,7 @@ pub enum FormatMetaVersion {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum FormatBackend {
|
||||
#[serde(rename = "xl")]
|
||||
Erasure,
|
||||
@@ -64,7 +64,7 @@ pub struct FormatErasureV3 {
|
||||
pub distribution_algo: DistributionAlgoVersion,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum FormatErasureVersion {
|
||||
#[serde(rename = "1")]
|
||||
V1,
|
||||
@@ -77,7 +77,7 @@ pub enum FormatErasureVersion {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum DistributionAlgoVersion {
|
||||
#[serde(rename = "CRCMOD")]
|
||||
V1,
|
||||
@@ -121,6 +121,15 @@ pub struct FormatV3 {
|
||||
pub disk_info: Option<DiskInfo>,
|
||||
}
|
||||
|
||||
pub(crate) type SharedFormatIdentity<'a> = (
|
||||
&'a FormatMetaVersion,
|
||||
&'a FormatBackend,
|
||||
&'a Uuid,
|
||||
&'a FormatErasureVersion,
|
||||
&'a [Vec<Uuid>],
|
||||
&'a DistributionAlgoVersion,
|
||||
);
|
||||
|
||||
impl TryFrom<&[u8]> for FormatV3 {
|
||||
type Error = JsonError;
|
||||
|
||||
@@ -198,52 +207,24 @@ impl FormatV3 {
|
||||
}
|
||||
|
||||
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
|
||||
let mut tmp = other.clone();
|
||||
let this = tmp.erasure.this;
|
||||
tmp.erasure.this = Uuid::nil();
|
||||
|
||||
if self.erasure.sets.len() != other.erasure.sets.len() {
|
||||
return Err(Error::other(format!(
|
||||
"Expected number of sets {}, got {}",
|
||||
self.erasure.sets.len(),
|
||||
other.erasure.sets.len()
|
||||
)));
|
||||
if self.shared_identity() != other.shared_identity() {
|
||||
return Err(Error::other("storage formats do not match"));
|
||||
}
|
||||
|
||||
for i in 0..self.erasure.sets.len() {
|
||||
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
|
||||
return Err(Error::other(format!(
|
||||
"Each set should be of same size, expected {}, got {}",
|
||||
self.erasure.sets[i].len(),
|
||||
other.erasure.sets[i].len()
|
||||
)));
|
||||
}
|
||||
self.find_disk_index_by_disk_id(other.erasure.this).map(|_| ())
|
||||
}
|
||||
|
||||
for j in 0..self.erasure.sets[i].len() {
|
||||
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
|
||||
return Err(Error::other(format!(
|
||||
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
|
||||
i,
|
||||
j,
|
||||
self.erasure.sets[i][j].to_string(),
|
||||
other.erasure.sets[i][j].to_string(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..tmp.erasure.sets.len() {
|
||||
for j in 0..tmp.erasure.sets[i].len() {
|
||||
if this == tmp.erasure.sets[i][j] {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::other(format!(
|
||||
"DriveID {:?} not found in any drive sets {:?}",
|
||||
this, other.erasure.sets
|
||||
)))
|
||||
/// Fields that must agree across every disk in one erasure format,
|
||||
/// excluding the disk-specific `this` UUID and runtime-only `disk_info`.
|
||||
pub(crate) fn shared_identity(&self) -> SharedFormatIdentity<'_> {
|
||||
(
|
||||
&self.version,
|
||||
&self.format,
|
||||
&self.id,
|
||||
&self.erasure.version,
|
||||
&self.erasure.sets,
|
||||
&self.erasure.distribution_algo,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,6 +418,30 @@ mod test {
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_other_rejects_shared_identity_mismatches() {
|
||||
type FormatMutation = (&'static str, fn(&mut FormatV3));
|
||||
|
||||
let format = FormatV3::new(1, 2);
|
||||
let mutations: [FormatMutation; 5] = [
|
||||
("meta version", |other| other.version = FormatMetaVersion::Unknown),
|
||||
("backend", |other| other.format = FormatBackend::ErasureSingle),
|
||||
("deployment id", |other| other.id = Uuid::new_v4()),
|
||||
("erasure version", |other| other.erasure.version = FormatErasureVersion::V2),
|
||||
("distribution algorithm", |other| {
|
||||
other.erasure.distribution_algo = DistributionAlgoVersion::V2
|
||||
}),
|
||||
];
|
||||
|
||||
for (field, mutate) in mutations {
|
||||
let mut other = format.clone();
|
||||
other.erasure.this = format.erasure.sets[0][0];
|
||||
mutate(&mut other);
|
||||
|
||||
assert!(format.check_other(&other).is_err(), "{field} mismatch must be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_other_different_set_count() {
|
||||
let format1 = FormatV3::new(2, 4);
|
||||
|
||||
@@ -93,3 +93,6 @@ pub(crate) mod ecstore_validation_blackbox;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_metrics;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_tracing;
|
||||
|
||||
@@ -207,10 +207,6 @@ pub(crate) async fn ensure_boot_time() {
|
||||
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
|
||||
rustfs_common::get_global_init_time().await
|
||||
}
|
||||
|
||||
pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
|
||||
if is_erasure_sd().await {
|
||||
None
|
||||
@@ -428,7 +424,11 @@ pub(crate) async fn record_local_disk_id(instance_ctx: &Arc<InstanceContext>, di
|
||||
pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Option<Uuid>, endpoint: String) {
|
||||
let id_map = local_disk_id_map_handle();
|
||||
let mut disk_id_map = id_map.write().await;
|
||||
if let Some(previous_id) = previous {
|
||||
if let Some(previous_id) = previous
|
||||
&& disk_id_map
|
||||
.get(&previous_id)
|
||||
.is_some_and(|registered_endpoint| registered_endpoint == &endpoint)
|
||||
{
|
||||
disk_id_map.remove(&previous_id);
|
||||
}
|
||||
if let Some(current_id) = current {
|
||||
@@ -562,10 +562,14 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{LockRegistry, local_node_name, set_local_node_name};
|
||||
use super::{
|
||||
LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, replace_local_disk_id,
|
||||
set_local_node_name,
|
||||
};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use rustfs_lock::{LocalClient, LockClient};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn url_endpoint(raw: &str) -> Endpoint {
|
||||
Endpoint {
|
||||
@@ -611,4 +615,17 @@ mod tests {
|
||||
|
||||
assert_eq!(observed, next);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn clearing_a_stale_disk_id_does_not_remove_another_endpoint() {
|
||||
clear_local_disk_id_map_for_test().await;
|
||||
let disk_id = Uuid::new_v4();
|
||||
replace_local_disk_id(None, Some(disk_id), "endpoint-a".to_string()).await;
|
||||
|
||||
replace_local_disk_id(Some(disk_id), None, "endpoint-b".to_string()).await;
|
||||
|
||||
assert_eq!(local_disk_path_by_id(&disk_id).await, Some("endpoint-a".to_string()));
|
||||
clear_local_disk_id_map_for_test().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,6 +71,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
MadminScannerMetrics {
|
||||
collected_at: metrics.collected_at,
|
||||
current_cycle: metrics.current_cycle,
|
||||
current_cycle_active: Some(metrics.current_cycle_active),
|
||||
current_started: metrics.current_started,
|
||||
cycles_completed_at: metrics.cycles_completed_at,
|
||||
ongoing_buckets: metrics.ongoing_buckets,
|
||||
@@ -398,10 +399,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
|
||||
if types.contains(&MetricType::SCANNER) {
|
||||
debug!("start get scanner metrics");
|
||||
let mut metrics = global_metrics().report().await;
|
||||
if let Some(init_time) = runtime_sources::scanner_init_time().await {
|
||||
metrics.current_started = init_time;
|
||||
}
|
||||
let metrics = global_metrics().report().await;
|
||||
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
|
||||
}
|
||||
|
||||
@@ -540,7 +538,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use rustfs_common::metrics::CurrentCycle;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
@@ -588,7 +588,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
@@ -598,6 +601,8 @@ mod test {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
let usage = scanner
|
||||
@@ -608,6 +613,39 @@ mod test {
|
||||
assert_eq!(usage.cycles, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
next: 1,
|
||||
started: cycle_started,
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
|
||||
|
||||
let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
|
||||
|
||||
global_metrics()
|
||||
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
|
||||
.await;
|
||||
global_metrics().set_cycle(previous_cycle).await;
|
||||
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
|
||||
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
|
||||
let mut aggregated = RealtimeMetrics::default();
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_pacing_pressure() {
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
|
||||
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use std::collections::{HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
||||
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -91,6 +94,180 @@ lazy_static! {
|
||||
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
impl RemoteVersionStateFleetProof {
|
||||
fn token(&self) -> RemoteVersionStateFleetProofToken {
|
||||
RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||
peer_epochs: self.peer_epochs.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RemoteVersionStateFleetProofState {
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
topology_conflict: bool,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
|
||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
|
||||
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof_in(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
) {
|
||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||
}
|
||||
|
||||
fn publish_remote_version_state_probe_result(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
topology_fingerprint: &str,
|
||||
result: Result<BTreeMap<String, Uuid>>,
|
||||
observed_at: Instant,
|
||||
) -> Option<Error> {
|
||||
match result {
|
||||
Ok(peer_epochs) => {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let peer_epochs = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||
state.proof = Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
peer_epochs,
|
||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
});
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
replace_remote_version_state_fleet_proof_in(slot, None);
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn acquire_remote_version_state_fleet_proof_from(
|
||||
state: &RemoteVersionStateFleetProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<RemoteVersionStateFleetProofToken> {
|
||||
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
||||
return None;
|
||||
}
|
||||
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return false;
|
||||
}
|
||||
state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == *expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& Instant::now() < current.expires_at
|
||||
})
|
||||
}
|
||||
|
||||
fn remote_version_state_fleet_proof_valid_at(
|
||||
proof: Option<&RemoteVersionStateFleetProof>,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
let mut state = remote_version_state_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
state.proof = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
replace_remote_version_state_fleet_proof(None);
|
||||
} else if let Some(err) = publish_remote_version_state_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
result,
|
||||
Instant::now(),
|
||||
) {
|
||||
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
@@ -115,7 +292,17 @@ pub struct NotificationSys {
|
||||
|
||||
impl NotificationSys {
|
||||
pub async fn new(eps: EndpointServerPools) -> Self {
|
||||
let expected_remote_hosts = eps
|
||||
.peer_grid_host_slots_sorted()
|
||||
.into_iter()
|
||||
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
|
||||
.collect::<Vec<_>>();
|
||||
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
|
||||
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
|
||||
expected_remote_hosts
|
||||
} else {
|
||||
peer_topology_hosts
|
||||
};
|
||||
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
|
||||
Self {
|
||||
peer_clients,
|
||||
@@ -125,6 +312,24 @@ impl NotificationSys {
|
||||
tier_config_reload_workers: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("remote version state capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
|
||||
client.probe_remote_version_state(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NotificationPeerErr {
|
||||
@@ -1344,8 +1549,11 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
}
|
||||
TierConfigReloadFinish::Pending => retry_attempt = 0,
|
||||
},
|
||||
TierConfigReloadOutcome::Terminal(_) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadOutcome::Terminal(err) => match sys.finish_tier_config_reload_worker(&host) {
|
||||
TierConfigReloadFinish::Completed => {
|
||||
// This peer keeps the previous tier configuration for good, so record
|
||||
// why. Dropping the error here hides the only evidence of a divergent
|
||||
// node behind an outcome label that cannot be acted on.
|
||||
warn!(
|
||||
event = EVENT_NOTIFICATION_PEER_PROPAGATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1353,6 +1561,7 @@ async fn run_tier_config_reload_worker<F, Fut>(
|
||||
action = "reload_transition_tier_config",
|
||||
host,
|
||||
outcome = "terminal",
|
||||
error = ?err,
|
||||
"tier configuration reload stopped after a terminal outcome"
|
||||
);
|
||||
return;
|
||||
@@ -1835,15 +2044,10 @@ fn synthesized_disks(host: &str, endpoints: &EndpointServerPools, state: ItemSta
|
||||
/// Whether `peer_host` refers to the same node as an endpoint whose
|
||||
/// `host_port()` is `ep_host_port`.
|
||||
///
|
||||
/// `PeerRestClient::host` is an `XHost`, which resolves names to an address on
|
||||
/// construction (`hosts_sorted` -> `XHost::try_from` -> `to_socket_addrs`), so
|
||||
/// `peer_host` is the resolved `IP:port`. An endpoint's `host_port()`, however,
|
||||
/// is `url.host():port` — still the raw `hostname:port` on hostname-based
|
||||
/// deployments. A plain string compare therefore misses on hostname clusters,
|
||||
/// leaving the synthesized/degraded drive list empty and `unknownDisks` at 0
|
||||
/// (rustfs/rustfs#4607 follow-up). Compare directly first (fast path / IP
|
||||
/// deployments), then canonicalize the endpoint side through the same `XHost`
|
||||
/// resolution and compare again.
|
||||
/// Current topology clients preserve the endpoint `hostname:port`, so the
|
||||
/// direct comparison is the normal path. The resolution fallback keeps
|
||||
/// compatibility with older or manually constructed clients whose `XHost`
|
||||
/// contains a resolved `IP:port` (rustfs/rustfs#4607 follow-up).
|
||||
fn endpoint_host_matches(peer_host: &str, ep_host_port: &str) -> bool {
|
||||
if peer_host == ep_host_port {
|
||||
return true;
|
||||
@@ -1886,6 +2090,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
|
||||
let now = Instant::now();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(peer_epochs),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
|
||||
assert!(peer_epochs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let captured = proof.token();
|
||||
let restarted = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: proof.expires_at,
|
||||
};
|
||||
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let epoch = Uuid::new_v4();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
||||
let original = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("successful probe should publish proof")
|
||||
.token();
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let renewed = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("renewal should retain proof")
|
||||
.token();
|
||||
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
|
||||
|
||||
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
|
||||
.is_none()
|
||||
);
|
||||
let replaced = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("restarted peer should publish a new proof")
|
||||
.token();
|
||||
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
let mut state = RemoteVersionStateFleetProofState {
|
||||
proof: Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
}),
|
||||
topology_conflict: false,
|
||||
};
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
|
||||
state.topology_conflict = true;
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
|
||||
.expect("first member should be admitted");
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
||||
);
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
||||
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable configured member must fail the fleet proof");
|
||||
assert!(err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing configured member slot must fail the fleet proof");
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -72,6 +72,7 @@ use crate::{
|
||||
cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState},
|
||||
config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
|
||||
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
|
||||
layout::endpoints::EndpointServerPools,
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::get_lock_acquire_timeout,
|
||||
@@ -236,6 +237,7 @@ struct TierPublishTransition {
|
||||
|
||||
struct PreparedTierDriver {
|
||||
tier_name: String,
|
||||
tier_config: TierConfig,
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
@@ -903,14 +905,17 @@ async fn remote_tier_mutation_peers() -> io::Result<Vec<Arc<dyn TierMutationPeer
|
||||
let Some(endpoints) = runtime_sources::endpoint_pools() else {
|
||||
return Err(tier_mutation_replay_error("cluster endpoint topology is not initialized"));
|
||||
};
|
||||
let remote_host_count = endpoints.hosts_sorted().iter().flatten().count();
|
||||
let (peers, _) = PeerRestClient::new_clients(endpoints).await;
|
||||
remote_tier_mutation_peers_from_topology(endpoints).await
|
||||
}
|
||||
|
||||
async fn remote_tier_mutation_peers_from_topology(endpoints: EndpointServerPools) -> io::Result<Vec<Arc<dyn TierMutationPeer>>> {
|
||||
let (peers, _, remote_topology_hosts) = PeerRestClient::new_clients_with_topology(endpoints).await;
|
||||
let peers = peers
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|peer| Arc::new(peer) as Arc<dyn TierMutationPeer>)
|
||||
.collect::<Vec<_>>();
|
||||
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_host_count)?;
|
||||
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_topology_hosts.len())?;
|
||||
Ok(peers)
|
||||
}
|
||||
|
||||
@@ -1342,6 +1347,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
|
||||
|
||||
struct TierDriverGeneration {
|
||||
tier_name: Arc<str>,
|
||||
tier_config: TierConfig,
|
||||
generation: DriverRevision,
|
||||
// Process-local only: this may reflect credential changes and must never be persisted or logged.
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
@@ -1349,6 +1355,9 @@ struct TierDriverGeneration {
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
driver: SharedWarmBackend,
|
||||
reconciler: tokio::sync::OnceCell<
|
||||
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
|
||||
>,
|
||||
accepting: AtomicBool,
|
||||
active_leases: AtomicUsize,
|
||||
drained: tokio::sync::Notify,
|
||||
@@ -1473,6 +1482,35 @@ impl TierOperationLease {
|
||||
self.inner.backend_identity
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
transaction_id: uuid::Uuid,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
let Some(reconciler) = self
|
||||
.inner
|
||||
.reconciler
|
||||
.get_or_try_init(|| async {
|
||||
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
|
||||
.await
|
||||
.map(|reconciler| reconciler.map(Arc::from))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.message))?
|
||||
else {
|
||||
return self.inner.driver.probe_transition_candidate(object).await;
|
||||
};
|
||||
reconciler
|
||||
.probe_transition_candidate_for(
|
||||
object,
|
||||
crate::services::tier::warm_backend::TransitionCandidateIdentity {
|
||||
transaction_id,
|
||||
destination_id: self.backend_identity(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
|
||||
self.inner.driver.validate_remote_version_id(remote_version_id)?;
|
||||
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
|
||||
@@ -2833,6 +2871,7 @@ impl TierConfigMgr {
|
||||
let exact_get_delete = tier_exact_get_delete(config);
|
||||
Some(PreparedTierDriver {
|
||||
tier_name: tier_name.to_string(),
|
||||
tier_config: config.clone(),
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
@@ -2861,11 +2900,13 @@ impl TierConfigMgr {
|
||||
})?;
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(prepared.tier_name.as_str()),
|
||||
tier_config: prepared.tier_config.clone(),
|
||||
generation,
|
||||
config_fingerprint: prepared.config_fingerprint,
|
||||
backend_identity: prepared.backend_identity,
|
||||
exact_get_delete: prepared.exact_get_delete,
|
||||
driver: prepared.driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -3609,11 +3650,13 @@ impl TierConfigMgr {
|
||||
let driver: SharedWarmBackend = Arc::from(driver);
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(tier_name),
|
||||
tier_config: config.clone(),
|
||||
generation,
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
driver: driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -4268,6 +4311,34 @@ fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::layout::{
|
||||
endpoint::Endpoint,
|
||||
endpoints::{Endpoints, PoolEndpoints, SetupType},
|
||||
};
|
||||
|
||||
struct SetupTypeGuard {
|
||||
previous: SetupType,
|
||||
}
|
||||
|
||||
impl SetupTypeGuard {
|
||||
async fn switch_to(next: SetupType) -> Self {
|
||||
let previous = runtime_sources::current_setup_type().await;
|
||||
runtime_sources::set_setup_type(next).await;
|
||||
Self { previous }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SetupTypeGuard {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.previous.clone();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::block_in_place(|| {
|
||||
handle.block_on(async move {
|
||||
runtime_sources::set_setup_type(previous).await;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn build_s3_tier(name: &str) -> TierConfig {
|
||||
TierConfig {
|
||||
@@ -6315,6 +6386,42 @@ mod tests {
|
||||
assert!(err.to_string().contains("without peer commit clients"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
async fn tier_mutation_peer_composition_preserves_unresolved_topology_slots() {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let mut endpoint = Endpoint::try_from(format!("http://rustfs-{disk_index}.invalid:9000/data{disk_index}").as_str())
|
||||
.expect("unresolved topology endpoint should parse without DNS");
|
||||
endpoint.is_local = disk_index == 0;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let topology = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "unresolved-tier-mutation-topology".to_string(),
|
||||
platform: "test".to_string(),
|
||||
}]);
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let peers = remote_tier_mutation_peers_from_topology(topology)
|
||||
.await
|
||||
.expect("every unresolved remote topology slot should retain a tier mutation client");
|
||||
assert_eq!(
|
||||
peers.iter().map(|peer| peer.peer_label()).collect::<Vec<_>>(),
|
||||
vec![
|
||||
"http://rustfs-1.invalid:9000".to_string(),
|
||||
"http://rustfs-2.invalid:9000".to_string(),
|
||||
"http://rustfs-3.invalid:9000".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_fanout_prepare_failure_aborts_prepared_peers_without_cas() {
|
||||
let manager = TierConfigMgr::new();
|
||||
|
||||
@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TransitionCandidateIdentity {
|
||||
pub transaction_id: uuid::Uuid,
|
||||
pub destination_id: [u8; 32],
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TransitionCandidateReconciler {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
metadata.remove(key);
|
||||
}
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
for key in [
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
|
||||
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
|
||||
] {
|
||||
if let Some(value) = metadata.remove(&key) {
|
||||
metadata.insert(format!("x-amz-meta-{key}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
pub(crate) async fn new_transition_candidate_reconciler(
|
||||
tier: &TierConfig,
|
||||
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
|
||||
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
|
||||
TierType::S3 => Box::new(
|
||||
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::MinIO => Box::new(
|
||||
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::RustFS => Box::new(
|
||||
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::R2 => Box::new(
|
||||
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
Ok(Some(reconciler))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -775,6 +849,37 @@ mod tests {
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"5a".repeat(32),
|
||||
);
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
|
||||
)));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}{suffix}",
|
||||
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -45,6 +46,19 @@ const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let generation = remote_version
|
||||
.parse::<i64>()
|
||||
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
|
||||
if generation <= 0 {
|
||||
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
|
||||
}
|
||||
Ok(Some(generation))
|
||||
}
|
||||
|
||||
pub struct WarmBackendGCS {
|
||||
pub client: Arc<Storage>,
|
||||
pub control: Arc<StorageControl>,
|
||||
@@ -105,6 +119,10 @@ impl WarmBackendGCS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendGCS {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
parse_generation(remote_version_id).map(|_| ())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -135,6 +153,9 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
|
||||
// Honor the requested byte range so Range GETs on tiered objects return the exact
|
||||
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
|
||||
@@ -164,13 +185,15 @@ impl WarmBackend for WarmBackendGCS {
|
||||
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
|
||||
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
|
||||
// remote data forever.
|
||||
self.control
|
||||
let mut req = self
|
||||
.control
|
||||
.delete_object()
|
||||
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
|
||||
.set_object(self.get_dest(object))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
.set_object(self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -191,6 +214,30 @@ impl WarmBackend for WarmBackendGCS {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_generation;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
#[test]
|
||||
fn generation_parser_preserves_exact_numeric_versions() {
|
||||
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
|
||||
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
|
||||
assert_eq!(
|
||||
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
|
||||
Some(i64::MAX)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_parser_rejects_unknown_or_non_positive_versions() {
|
||||
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
|
||||
let err = parse_generation(value).expect_err("unknown generation must fail closed");
|
||||
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::client::{
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -36,7 +37,10 @@ use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -189,44 +193,188 @@ impl WarmBackendS3 {
|
||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||
}
|
||||
|
||||
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
|
||||
async fn probe_transition_candidate_versions(
|
||||
&self,
|
||||
object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &self.get_dest(object));
|
||||
opts.set("max-keys", "2");
|
||||
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
candidates.extend(&remote_object, &versions);
|
||||
if candidates.is_ambiguous() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut matched_version = None;
|
||||
let mut saw_unproven_candidate = false;
|
||||
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
matched_version = Some(version.version_id.clone());
|
||||
} else {
|
||||
saw_unproven_candidate = true;
|
||||
}
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
if matched_version.is_none() && saw_unproven_candidate {
|
||||
return Ok(TransitionCandidateProbe::Unsupported);
|
||||
}
|
||||
let candidates = TransitionCandidateVersions {
|
||||
version_id: matched_version,
|
||||
ambiguous: false,
|
||||
};
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_transition_candidate_versions(
|
||||
remote_object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
versions: &ListVersionsResult,
|
||||
) -> TransitionCandidateProbe {
|
||||
if versions.is_truncated {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
|
||||
let Some(version) = exact_versions.next() else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
fn transition_candidate_metadata_matches(
|
||||
metadata: &HashMap<String, String>,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
|
||||
};
|
||||
if exact_versions.next().is_some() {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
|
||||
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
|
||||
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
|
||||
if transaction_id.is_none() || destination_id.is_none() {
|
||||
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|
||||
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"transition candidate identity metadata is empty or conflicting",
|
||||
));
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
let expected_transaction_id = identity.transaction_id.to_string();
|
||||
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
|
||||
}
|
||||
|
||||
fn classify_transition_candidates(
|
||||
candidates: TransitionCandidateVersions,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let probe = candidates.classify(bucket_versioning);
|
||||
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
|
||||
validate_remote_version_id(version_id)?;
|
||||
}
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
fn advance_version_markers(
|
||||
key_marker: &mut String,
|
||||
version_id_marker: &mut String,
|
||||
versions: &ListVersionsResult,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
|
||||
if next_markers == (&*key_marker, &*version_id_marker) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"ListObjectVersions pagination markers did not advance",
|
||||
));
|
||||
}
|
||||
key_marker.clone_from(&versions.next_key_marker);
|
||||
version_id_marker.clone_from(&versions.next_version_id_marker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TransitionCandidateVersions {
|
||||
version_id: Option<String>,
|
||||
ambiguous: bool,
|
||||
}
|
||||
|
||||
impl TransitionCandidateVersions {
|
||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
if self.version_id.is_some() {
|
||||
self.ambiguous = true;
|
||||
return;
|
||||
}
|
||||
self.version_id = Some(version.version_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
fn is_ambiguous(&self) -> bool {
|
||||
self.ambiguous
|
||||
}
|
||||
|
||||
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
||||
if self.ambiguous {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
let Some(version_id) = self.version_id else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,74 +423,163 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
for page in pages {
|
||||
candidates.extend("archive/object", page);
|
||||
}
|
||||
candidates.classify(bucket_versioning)
|
||||
}
|
||||
|
||||
fn candidate_identity() -> TransitionCandidateIdentity {
|
||||
TransitionCandidateIdentity {
|
||||
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
|
||||
destination_id: [0x5a; 32],
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
identity.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity.destination_id),
|
||||
);
|
||||
metadata
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_requires_exact_compatible_metadata() {
|
||||
let identity = candidate_identity();
|
||||
let metadata = candidate_metadata(identity);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
|
||||
|
||||
let mut adjacent = metadata;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut adjacent,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
|
||||
|
||||
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
|
||||
let identity = candidate_identity();
|
||||
let mut metadata = candidate_metadata(identity);
|
||||
metadata.insert(
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
),
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_classifier_is_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[], &[], false),
|
||||
),
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], false),
|
||||
&[list_versions(&[("archive/object", "version-a")], &[], false)],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Suspended,
|
||||
&list_versions(&[("archive/object", "null")], &[], false),
|
||||
&[list_versions(&[("archive/object", "null")], &[], false)],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("null".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
|
||||
&[list_versions(
|
||||
&[("archive/object", "version-a"), ("archive/object", "version-b")],
|
||||
&[],
|
||||
false,
|
||||
)],
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[], &[("archive/object", "marker-a")], true),
|
||||
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
|
||||
],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
&[
|
||||
list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
list_versions(&[("archive/object", "version-b")], &[], false),
|
||||
],
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_pagination_advances_both_markers() {
|
||||
let mut key_marker = "old-key".to_string();
|
||||
let mut version_id_marker = "old-version".to_string();
|
||||
let page = ListVersionsResult {
|
||||
next_key_marker: "next-key".to_string(),
|
||||
next_version_id_marker: "next-version".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect("new ListObjectVersions markers should advance pagination");
|
||||
assert_eq!(key_marker, "next-key");
|
||||
assert_eq!(version_id_marker, "next-version");
|
||||
|
||||
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect_err("repeated ListObjectVersions markers must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_rejects_untrusted_version_ids() {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
|
||||
|
||||
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
|
||||
.expect_err("control characters in listed version IDs must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_bucket_versioning_status_parser_fails_closed() {
|
||||
assert_eq!(
|
||||
@@ -397,12 +634,7 @@ impl WarmBackend for WarmBackendS3 {
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
let versions = self.list_transition_candidate_versions(object).await?;
|
||||
Ok(classify_transition_candidate_versions(
|
||||
&self.get_dest(object),
|
||||
bucket_versioning,
|
||||
&versions,
|
||||
))
|
||||
self.probe_transition_candidate_versions(object, bucket_versioning).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
@@ -414,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, part_transaction_path,
|
||||
@@ -3952,9 +3953,9 @@ impl SetDisks {
|
||||
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
|
||||
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
|
||||
/// * If a disk holds the object directory but its `xl.meta` is missing or
|
||||
/// unparsable, the object is treated as degraded and NOTHING is removed —
|
||||
/// the unreadable copy could be the only one naming a live data dir, and a
|
||||
/// heal must run first.
|
||||
/// unparsable, the object is treated as degraded and unmarked data dirs are
|
||||
/// never removed. A data dir carrying a committed delete-transaction marker
|
||||
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
|
||||
/// * Only subdirectories whose names parse as a UUID are ever considered;
|
||||
/// removal is non-recursive-safe via a recursive delete of the full stray
|
||||
/// data-dir path only.
|
||||
@@ -3969,7 +3970,7 @@ impl SetDisks {
|
||||
// physical UUID subdirectories present on each disk. Abort on any degraded
|
||||
// copy so a healable object is never stripped of a referenced data dir.
|
||||
let mut referenced: HashSet<Uuid> = HashSet::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
|
||||
let mut healthy_metas = 0usize;
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
@@ -4005,6 +4006,22 @@ impl SetDisks {
|
||||
// to the orphan-dir / dangling-object heal paths.
|
||||
continue;
|
||||
}
|
||||
let mut committed = Vec::with_capacity(physical.len());
|
||||
for dir in physical {
|
||||
let data_dir = format!("{object}/{dir}");
|
||||
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
|
||||
entries.iter().any(|entry| {
|
||||
entry
|
||||
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
|
||||
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
|
||||
})
|
||||
});
|
||||
committed.push((dir, committed_delete));
|
||||
}
|
||||
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
|
||||
per_disk_dirs.push((i, committed));
|
||||
continue;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
bucket, object,
|
||||
@@ -4041,22 +4058,16 @@ impl SetDisks {
|
||||
|
||||
healthy_metas += 1;
|
||||
if !physical.is_empty() {
|
||||
per_disk_dirs.push((i, physical));
|
||||
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
|
||||
}
|
||||
}
|
||||
|
||||
// No healthy metadata anywhere: this is not a live object, so surplus dirs
|
||||
// (if any) belong to the dangling-object heal path, not here.
|
||||
if healthy_metas == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Phase 2: delete every physical data dir not referenced by the union.
|
||||
let mut removed = 0usize;
|
||||
for (i, physical) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
for dir in physical {
|
||||
if referenced.contains(&dir) {
|
||||
for (dir, committed_delete) in physical {
|
||||
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
|
||||
@@ -555,7 +555,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
Self::update_file_info_quorum_hash(&mut hasher, meta);
|
||||
let digest = hasher.finalize();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -343,20 +343,23 @@ impl SetDisks {
|
||||
}
|
||||
};
|
||||
|
||||
// The drive's format may place it in a different erasure set than this
|
||||
// one. Claiming a misplaced drive into `self.disks` would let two sets
|
||||
// manage the same drive and degrade together, so reject it here
|
||||
// (backlog#799 B19).
|
||||
if set_idx != self.set_index {
|
||||
// Claiming a misplaced drive into `self.disks` would let two slots or
|
||||
// sets manage the same drive and degrade together (backlog#799 B19).
|
||||
if set_idx != self.set_index || self.set_endpoints.get(disk_idx) != Some(ep) {
|
||||
warn!(
|
||||
"renew_disk: drive {:?} belongs to set {} but is being renewed on set {}; skipping",
|
||||
ep, set_idx, self.set_index
|
||||
endpoint = %ep,
|
||||
format_set_index = set_idx,
|
||||
format_disk_index = disk_idx,
|
||||
endpoint_pool_index = ep.pool_idx,
|
||||
endpoint_set_index = ep.set_idx,
|
||||
endpoint_disk_index = ep.disk_idx,
|
||||
expected_pool_index = self.pool_index,
|
||||
expected_set_index = self.set_index,
|
||||
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check that the endpoint matches
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
new_disk.enable_health_check();
|
||||
|
||||
@@ -715,6 +718,108 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
|
||||
let disk_count = 3;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut temp_dirs = Vec::with_capacity(disk_count);
|
||||
let mut endpoints = Vec::with_capacity(disk_count);
|
||||
let mut fixture_disks = Vec::with_capacity(disk_count);
|
||||
|
||||
for disk_idx in 0..disk_count {
|
||||
let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
fixture_disks.push(disk);
|
||||
}
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![Some(fixture_disks[0].clone()), None, None])),
|
||||
disk_count,
|
||||
disk_count / 2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut other_cluster_format = format.clone();
|
||||
other_cluster_format.id = Uuid::new_v4();
|
||||
other_cluster_format.erasure.this = format.erasure.sets[0][2];
|
||||
save_format_file(&Some(fixture_disks[2].clone()), &Some(other_cluster_format))
|
||||
.await
|
||||
.expect("other-cluster format should be written for the rejection test");
|
||||
|
||||
set_disks.renew_disk(&endpoints[2]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert!(
|
||||
disks[2].is_none(),
|
||||
"a disk from another deployment must remain detached even when its slot UUID matches"
|
||||
);
|
||||
|
||||
let mut correct_format = format.clone();
|
||||
correct_format.erasure.this = format.erasure.sets[0][2];
|
||||
let replacement_disk = new_disk(
|
||||
&endpoints[2],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("third endpoint should reopen after other-cluster rejection");
|
||||
save_format_file(&Some(replacement_disk), &Some(correct_format))
|
||||
.await
|
||||
.expect("correct slot format should be restored");
|
||||
|
||||
set_disks.renew_disk(&endpoints[2]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert_eq!(disks[2].as_ref().expect("the restored third slot should attach").endpoint(), endpoints[2]);
|
||||
|
||||
let third_disk = disks[2].clone();
|
||||
let mut wrong_slot_format = format.clone();
|
||||
wrong_slot_format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&third_disk, &Some(wrong_slot_format))
|
||||
.await
|
||||
.expect("wrong-slot format should be written for the rejection test");
|
||||
set_disks.disks.write().await[2] = None;
|
||||
|
||||
let mut misplaced_endpoint = endpoints[2].clone();
|
||||
misplaced_endpoint.set_disk_index(0);
|
||||
set_disks.renew_disk(&misplaced_endpoint).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert!(disks[2].is_none(), "a disk claiming another endpoint's slot must remain detached");
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
// SetDisks split P0 (#816): the borrow handle must mirror the core state and
|
||||
// the List operation family must run identically through it.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
@@ -36,6 +36,7 @@ const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum MultipartCommitPause {
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
@@ -47,9 +48,10 @@ struct MultipartCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause: MultipartCommitPause,
|
||||
armed: AtomicBool,
|
||||
expected_arrivals: usize,
|
||||
arrivals: AtomicUsize,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
release: tokio::sync::Semaphore,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -64,13 +66,24 @@ static MULTIPART_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc
|
||||
#[cfg(test)]
|
||||
impl MultipartCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str, pause: MultipartCommitPause) -> Self {
|
||||
Self::install_for_arrivals(bucket, object, pause, 1)
|
||||
}
|
||||
|
||||
pub(crate) fn install_for_arrivals(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pause: MultipartCommitPause,
|
||||
expected_arrivals: usize,
|
||||
) -> Self {
|
||||
assert!(expected_arrivals > 0, "multipart commit barrier must wait for at least one arrival");
|
||||
let state = Arc::new(MultipartCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause,
|
||||
armed: AtomicBool::new(true),
|
||||
expected_arrivals,
|
||||
arrivals: AtomicUsize::new(0),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Semaphore::new(0),
|
||||
});
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -83,20 +96,28 @@ impl MultipartCommitBarrier {
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let arrived = self.state.arrived.notified();
|
||||
if self.state.arrivals.load(Ordering::Acquire) >= self.state.expected_arrivals {
|
||||
return;
|
||||
}
|
||||
arrived.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("multipart completion should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
self.state.release.add_permits(self.state.expected_arrivals);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for MultipartCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
self.release();
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -117,10 +138,20 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier
|
||||
&& barrier.armed.swap(false, Ordering::AcqRel)
|
||||
&& let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
(current < barrier.expected_arrivals).then_some(current + 1)
|
||||
})
|
||||
{
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
let arrival = previous + 1;
|
||||
if arrival == barrier.expected_arrivals {
|
||||
barrier.arrived.notify_one();
|
||||
}
|
||||
barrier
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("multipart commit barrier should remain open")
|
||||
.forget();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -693,6 +724,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
|
||||
// Serialize only the commit (rename_part), not the whole upload. Each
|
||||
// concurrent stream writes to its own unique temp dir (see `tmp_part`
|
||||
// above), so the encode/stream phase never conflicts and must stay
|
||||
@@ -3346,85 +3379,98 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_holds_object_then_upload_lock_through_commit() {
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"abort should return InvalidUploadID after completion, got {abort_err:?}"
|
||||
);
|
||||
let list_err = list
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
let list_err = list
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
|
||||
})
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(
|
||||
matches!(list_err, StorageError::InvalidUploadID(..)),
|
||||
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
@@ -550,13 +550,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
&self.ctx.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
gr,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
|
||||
}
|
||||
|
||||
// App-layer object data cache probe: metadata (etag/size) is resolved
|
||||
@@ -683,6 +677,18 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let snapshot_lease = if lock_optimization_enabled {
|
||||
match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
|
||||
Some(data_dir) => {
|
||||
let data_dir_path = format!("{object}/{data_dir}");
|
||||
acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
match codec_streaming_gate.decision {
|
||||
GetCodecStreamingDecision::Use => {
|
||||
match Self::get_object_decode_reader_with_fileinfo(
|
||||
@@ -711,13 +717,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
// Carry the hook probe result so the app layer skips its
|
||||
// now-redundant lookup on the streaming miss path (ODC-16).
|
||||
reader.body_source = body_source;
|
||||
return Ok(finish_set_disk_read_lock(
|
||||
reader,
|
||||
read_lock_guard.take(),
|
||||
lock_optimization_enabled,
|
||||
bucket,
|
||||
object,
|
||||
));
|
||||
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
|
||||
}
|
||||
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
|
||||
record_get_codec_streaming_gate_decision(
|
||||
@@ -756,15 +756,22 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let set_index = self.set_index;
|
||||
let pool_index = self.pool_index;
|
||||
let skip_verify = opts.skip_verify_bitrot;
|
||||
if lock_optimization_enabled {
|
||||
let producer_snapshot_lease = snapshot_lease.clone();
|
||||
if let Some(lease) = snapshot_lease {
|
||||
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
|
||||
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
|
||||
reader.stream = Box::new(SnapshotLeaseReader {
|
||||
inner: reader.stream,
|
||||
lease: Some(lease),
|
||||
terminal_error: false,
|
||||
});
|
||||
debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
|
||||
}
|
||||
|
||||
// When lock optimization is disabled, keep the read-lock guard in the
|
||||
// task so it lives for the duration of the streaming read.
|
||||
// The producer shares the lease lifetime with the body so cancellation
|
||||
// cannot release the snapshot while the duplex task is still unwinding.
|
||||
tokio::spawn(async move {
|
||||
let _guard = read_lock_guard;
|
||||
let _snapshot_lease = producer_snapshot_lease;
|
||||
let mut writer = GetObjectDownstreamWriter::new(wd);
|
||||
// Do not wrap the entire read+write pipeline in `disk_read_timeout`.
|
||||
// `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
|
||||
@@ -2313,25 +2320,78 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn persisted_transition_version(
|
||||
remote_version: &str,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn remote_version_state_writer_enabled() -> bool {
|
||||
remote_version_state_writer_fleet_proof().is_some()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
|
||||
remote_version_state_writer_requested()
|
||||
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
remote_version_state_writer_enabled_for(
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
),
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches(
|
||||
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
|
||||
) -> bool {
|
||||
remote_version_state_writer_fleet_proof_matches_for(
|
||||
remote_version_state_writer_requested(),
|
||||
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
|
||||
requested && fleet_proof_matches
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
|
||||
requested && fleet_confirmed && fleet_proof_valid
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
remote_version: &str,
|
||||
remote_version_state_writer_enabled: bool,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
|
||||
}
|
||||
let version_id = Uuid::parse_str(remote_version).map_err(|_| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the cluster capability gate",
|
||||
)
|
||||
})?;
|
||||
if version_id.is_nil() {
|
||||
return Err(std::io::Error::new(
|
||||
|
||||
match Uuid::parse_str(remote_version) {
|
||||
Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier returned a nil object version ID",
|
||||
));
|
||||
)),
|
||||
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the operator-attested live fleet capability gate",
|
||||
)),
|
||||
Err(_) if remote_version == "null" => {
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
|
||||
}
|
||||
Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
}
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2569,7 +2629,10 @@ mod transition_upload_completion_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod transition_version_id_tests {
|
||||
use super::{TransitionUploadCandidate, persisted_transition_version};
|
||||
use super::{
|
||||
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
|
||||
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
|
||||
};
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -2608,6 +2671,64 @@ mod transition_version_id_tests {
|
||||
"opaque-version-token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
|
||||
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
|
||||
("old defaults", false, false, false, false),
|
||||
("missing fleet confirmation", true, false, true, false),
|
||||
("missing local opt-in", false, true, true, false),
|
||||
("missing fleet proof", true, true, false, false),
|
||||
("explicitly unconfirmed fleet", true, false, true, false),
|
||||
("rolled-back writer", false, true, true, false),
|
||||
("fully upgraded fleet", true, true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
|
||||
for (case, requested, fleet_proof_matches, expected) in [
|
||||
("operator gate closed", false, true, false),
|
||||
("fleet proof changed", true, false, false),
|
||||
("current authorization", true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_gate_enables_null_and_opaque_remote_version_states() {
|
||||
for (remote_version, expected) in [
|
||||
("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
|
||||
(
|
||||
"opaque-version-token",
|
||||
(Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
persisted_transition_version_with_gate(remote_version, false).is_err(),
|
||||
"missing fleet confirmation must reject {remote_version:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
@@ -3766,6 +3887,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let dest_obj = transaction.remote_object.clone();
|
||||
let mut transition_meta = (*oi.user_defined).clone();
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
transaction.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
|
||||
);
|
||||
|
||||
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
|
||||
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
|
||||
@@ -3876,20 +4007,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
let fleet_proof = remote_version_state_writer_fleet_proof();
|
||||
let remote_version_requires_fleet_proof =
|
||||
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
|
||||
let (transition_version_id, transition_version_state) =
|
||||
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
};
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4005,6 +4140,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
|
||||
// This check is the fleet-proof lease linearization point. Revocation
|
||||
// blocks later commits; an already-authorized local quorum commit is
|
||||
// allowed to finish without holding a synchronous lock across I/O.
|
||||
if remote_version_requires_fleet_proof
|
||||
&& !fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(remote_version_state_writer_fleet_proof_matches)
|
||||
{
|
||||
drop(transition_lock_guard);
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::other("remote version state fleet capability changed during transition"));
|
||||
}
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -7166,7 +7316,6 @@ mod transition_source_identity_matrix_tests {
|
||||
async fn transition_source_identity_field_matrix_rejects_single_field_drift() {
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum IdentityField {
|
||||
VersionId,
|
||||
DataDir,
|
||||
ModTime,
|
||||
Size,
|
||||
@@ -7182,7 +7331,6 @@ mod transition_source_identity_matrix_tests {
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
|
||||
for (index, field) in [
|
||||
IdentityField::VersionId,
|
||||
IdentityField::DataDir,
|
||||
IdentityField::ModTime,
|
||||
IdentityField::Size,
|
||||
@@ -7236,10 +7384,6 @@ mod transition_source_identity_matrix_tests {
|
||||
|
||||
let mut changed = source.clone();
|
||||
match field {
|
||||
IdentityField::VersionId => {
|
||||
changed.version_id = Some(Uuid::new_v4());
|
||||
changed.fresh = true;
|
||||
}
|
||||
IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
|
||||
IdentityField::ModTime => {
|
||||
changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
|
||||
@@ -7280,15 +7424,12 @@ mod transition_source_identity_matrix_tests {
|
||||
);
|
||||
|
||||
match field {
|
||||
IdentityField::VersionId => assert_ne!(source.version_id, persisted.version_id),
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, persisted.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
|
||||
}
|
||||
if !matches!(field, IdentityField::VersionId) {
|
||||
assert_eq!(source.version_id, persisted.version_id);
|
||||
}
|
||||
assert_eq!(source.version_id, persisted.version_id);
|
||||
if !matches!(field, IdentityField::DataDir) {
|
||||
assert_eq!(source.data_dir, persisted.data_dir);
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
async fn await_bucket_namespace_operation<T, F>(
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
@@ -310,12 +310,13 @@ impl ECStore {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.lock_enabled {
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
meta.object_lock_config_xml =
|
||||
crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
if opts.versioning_enabled {
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
}
|
||||
|
||||
await_bucket_namespace_operation(
|
||||
@@ -568,6 +569,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::error::StorageError;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
@@ -589,6 +591,7 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Notify, OnceCell};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -1048,7 +1051,7 @@ mod tests {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
for disk_path in &disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
|
||||
.await
|
||||
.expect("empty directory remnant should be created");
|
||||
}
|
||||
@@ -1257,6 +1260,55 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
|
||||
let (_, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
|
||||
let object = "committed-after-empty-scan";
|
||||
let payload = b"object committed after DeleteBucket empty scan".to_vec();
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let barrier = install_delete_bucket_empty_scan_barrier();
|
||||
let delete_store = ecstore.clone();
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
ecstore
|
||||
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
|
||||
|
||||
barrier.release();
|
||||
let err = delete
|
||||
.await
|
||||
.expect("DeleteBucket task should join")
|
||||
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
|
||||
let mut reader = ecstore
|
||||
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object should remain readable after DeleteBucket fails");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object body should remain readable");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_recreation_does_not_publish_unverified_usage() {
|
||||
|
||||
@@ -30,6 +30,7 @@ impl ECStore {
|
||||
};
|
||||
|
||||
let mut count_no_heal = 0;
|
||||
let mut first_error = None;
|
||||
for pool in self.pools.iter() {
|
||||
let (mut result, err) = pool.heal_format(dry_run).await?;
|
||||
if let Some(err) = err {
|
||||
@@ -37,8 +38,8 @@ impl ECStore {
|
||||
StorageError::NoHealRequired => {
|
||||
count_no_heal += 1;
|
||||
}
|
||||
_ => {
|
||||
continue;
|
||||
err => {
|
||||
first_error.get_or_insert(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,6 +48,9 @@ impl ECStore {
|
||||
r.before.drives.append(&mut result.before.drives);
|
||||
r.after.drives.append(&mut result.after.drives);
|
||||
}
|
||||
if let Some(err) = first_error {
|
||||
return Ok((r, Some(err)));
|
||||
}
|
||||
if count_no_heal == self.pools.len() {
|
||||
info!(
|
||||
event = EVENT_HEAL_FORMAT_COMPLETED,
|
||||
@@ -165,3 +169,134 @@ impl ECStore {
|
||||
Err(StorageError::NotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::{DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
let mut foreign_format = canonical_format.clone();
|
||||
foreign_format.id = Uuid::new_v4();
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
|
||||
for disk_index in 0..3 {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
|
||||
.expect("temporary endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("temporary disk should open");
|
||||
let mut disk_format = foreign_format.clone();
|
||||
disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("foreign format should be written");
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "foreign-format-majority-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
};
|
||||
let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
|
||||
.await
|
||||
.expect("test pool should build around the cached canonical format");
|
||||
|
||||
let mut recoverable_format = FormatV3::new(1, 3);
|
||||
recoverable_format.id = canonical_format.id;
|
||||
let mut recoverable_temp_dirs = Vec::new();
|
||||
let mut recoverable_endpoints = Vec::new();
|
||||
let mut recoverable_disks = Vec::new();
|
||||
let mut unformatted_disk = None;
|
||||
for disk_index in 0..3 {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
|
||||
.expect("temporary endpoint should parse");
|
||||
endpoint.set_pool_index(1);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("temporary disk should open");
|
||||
if disk_index < 2 {
|
||||
let mut disk_format = recoverable_format.clone();
|
||||
disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("recoverable format should be written");
|
||||
} else {
|
||||
unformatted_disk = Some(disk.clone());
|
||||
}
|
||||
recoverable_temp_dirs.push(temp_dir);
|
||||
recoverable_endpoints.push(endpoint);
|
||||
recoverable_disks.push(Some(disk));
|
||||
}
|
||||
let recoverable_pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(recoverable_endpoints),
|
||||
cmd_line: "recoverable-format-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
};
|
||||
let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
|
||||
.await
|
||||
.expect("recoverable test pool should build");
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
|
||||
let store = ECStore {
|
||||
id: canonical_format.id,
|
||||
disk_map: HashMap::new(),
|
||||
pools: vec![pool, recoverable_pool],
|
||||
peer_sys: S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: RwLock::new(PoolMeta::default()),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
};
|
||||
|
||||
let (result, err) = store
|
||||
.handle_heal_format(false)
|
||||
.await
|
||||
.expect("format heal should return the typed pool error");
|
||||
assert!(
|
||||
matches!(err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign format majority must not be downgraded to a successful heal: {err:?}"
|
||||
);
|
||||
assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
|
||||
let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
|
||||
.await
|
||||
.expect("the later pool should be healed despite the first pool error");
|
||||
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +101,10 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
|
||||
matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
|
||||
}
|
||||
|
||||
fn should_retry_format_load(err: &Error) -> bool {
|
||||
!matches!(err, Error::CorruptedFormat)
|
||||
}
|
||||
|
||||
fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool {
|
||||
rebalance_meta_loaded && !decommission_running
|
||||
}
|
||||
@@ -294,7 +298,7 @@ impl ECStore {
|
||||
// periodic monitoring until format loading succeeds. Startup RPC
|
||||
// failures can still spawn recovery probes for peers that come up
|
||||
// after this node.
|
||||
let (disks, errs) = init_format::init_disks(
|
||||
let (mut disks, errs) = init_format::init_disks(
|
||||
&pool_eps.endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
@@ -311,7 +315,7 @@ impl ECStore {
|
||||
loop {
|
||||
match init_format::connect_load_init_formats(
|
||||
pool_first_is_local,
|
||||
&disks,
|
||||
&mut disks,
|
||||
pool_eps.set_count,
|
||||
pool_eps.drives_per_set,
|
||||
deployment_id,
|
||||
@@ -319,6 +323,7 @@ impl ECStore {
|
||||
.await
|
||||
{
|
||||
Ok(fm) => break Ok(fm),
|
||||
Err(e) if !should_retry_format_load(&e) => break Err(e),
|
||||
// Wrap the final error if we are giving up
|
||||
Err(e) if times >= 10 => {
|
||||
break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
|
||||
@@ -551,7 +556,7 @@ mod tests {
|
||||
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, load_pool_meta_for_startup, pool_first_endpoint_is_local,
|
||||
pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with,
|
||||
resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
|
||||
should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::{
|
||||
@@ -562,10 +567,12 @@ mod tests {
|
||||
},
|
||||
tier_sweeper::Jentry,
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
|
||||
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
|
||||
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
|
||||
save_transition_transaction_record,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
|
||||
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
|
||||
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, save_transition_transaction_record,
|
||||
},
|
||||
},
|
||||
client::transition_api::ReaderImpl,
|
||||
@@ -575,6 +582,7 @@ mod tests {
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{TIER_CONFIG_FILE, TierConfigMgr},
|
||||
tier_config::{TierConfig, TierType, TierWasabi},
|
||||
tier_mutation_intent::{
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
|
||||
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
|
||||
@@ -770,6 +778,13 @@ mod tests {
|
||||
assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_format_load_rejects_permanent_corruption() {
|
||||
assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
|
||||
assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
|
||||
assert!(should_retry_format_load(&StorageError::FirstDiskWait));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() {
|
||||
assert!(should_auto_start_rebalance_after_init(false, true));
|
||||
@@ -1163,6 +1178,37 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_transition_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier_name: &str,
|
||||
) -> MockWarmBackend {
|
||||
let backend = MockWarmBackend::new();
|
||||
let mut manager = handle.write().await;
|
||||
manager.tiers.insert(
|
||||
tier_name.to_string(),
|
||||
TierConfig {
|
||||
version: "v1".to_string(),
|
||||
tier_type: TierType::Wasabi,
|
||||
name: tier_name.to_string(),
|
||||
wasabi: Some(TierWasabi {
|
||||
name: tier_name.to_string(),
|
||||
endpoint: "https://s3.wasabisys.com".to_string(),
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
bucket: "mock-tier".to_string(),
|
||||
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
|
||||
region: "us-east-1".to_string(),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
manager
|
||||
.install_test_driver(tier_name, Box::new(backend.clone()))
|
||||
.expect("mock fallback tier driver should install");
|
||||
backend
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_tier_delete_journal_recovery(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
@@ -2738,7 +2784,7 @@ mod tests {
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
@@ -2809,6 +2855,157 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATOR";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("operator inspection should probe the candidate");
|
||||
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
|
||||
|
||||
let wrong_version = uuid::Uuid::new_v4().to_string();
|
||||
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
|
||||
.await
|
||||
.expect_err("a mismatched exact version must fail before deleting a candidate");
|
||||
assert!(matches!(
|
||||
err,
|
||||
TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected,
|
||||
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
|
||||
} if expected == wrong_version && observed == &remote_version
|
||||
));
|
||||
assert!(backend.contains(&transaction.remote_object).await);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("an incorrect exact version must retain the transaction journal");
|
||||
|
||||
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
|
||||
.await
|
||||
.expect("operator-confirmed exact candidate should be deleted");
|
||||
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
|
||||
assert!(result.journal_observed_after_delete);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("candidate deletion must retain the transaction journal");
|
||||
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("a separately confirmed missing candidate should permit finalization");
|
||||
assert!(matches!(
|
||||
load_transition_transaction_record(store, transaction.transaction_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_finalize_retains_record_without_missing_proof() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATORFAIL";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
|
||||
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
|
||||
));
|
||||
load_transition_transaction_record(store, transaction.transaction_id)
|
||||
.await
|
||||
.expect("an unsupported probe must retain the transaction journal");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -2819,7 +3016,7 @@ mod tests {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
|
||||
+1073
-136
File diff suppressed because it is too large
Load Diff
@@ -6895,6 +6895,7 @@ mod test {
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
disk.make_volume("bucket").await.expect("test bucket should be created");
|
||||
let mut fi = FileInfo::new("object", 1, 1);
|
||||
fi.volume = "bucket".to_owned();
|
||||
fi.name = "object".to_owned();
|
||||
|
||||
@@ -141,6 +141,7 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
mod heal;
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
@@ -427,11 +428,11 @@ impl ECStore {
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
static ref enableVersioningConfig: VersioningConfiguration = VersioningConfiguration {
|
||||
static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -988,7 +989,7 @@ mod tests {
|
||||
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
let (disks, errs) = init_disks(
|
||||
let (mut disks, errs) = init_disks(
|
||||
&endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
@@ -998,7 +999,7 @@ mod tests {
|
||||
.await;
|
||||
|
||||
assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
|
||||
connect_load_init_formats(true, &disks, 1, 4, None)
|
||||
connect_load_init_formats(true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("initialize format metadata");
|
||||
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Test-only guard against tracing's process-global callsite-interest cache.
|
||||
//!
|
||||
//! Any test that installs its own subscriber and then asserts on span or event
|
||||
//! context is asserting on process-global state. See
|
||||
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
|
||||
//! guard fixes it.
|
||||
|
||||
/// Stop a sibling test thread from poisoning tracing's process-global callsite
|
||||
/// interest cache while this test asserts on span or event context.
|
||||
///
|
||||
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
|
||||
/// the first thread to reach a callsite fixes that value. While at most one
|
||||
/// dispatcher is registered, tracing-core takes a fast path
|
||||
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
|
||||
/// newly-registered callsite's interest from whichever subscriber is current
|
||||
/// *on the registering thread*, and registration happens exactly once (guarded
|
||||
/// by a CAS in `DefaultCallsite::register`).
|
||||
///
|
||||
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
|
||||
/// reaches a production callsite first, from a thread with no subscriber
|
||||
/// installed: the interest is derived from that thread's `NoSubscriber` and
|
||||
/// cached as `Interest::never()` for the whole process. From then on the
|
||||
/// callsite is dead for *every* later caller, including a test that installed
|
||||
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
|
||||
/// a `warn!`/`info!` event never fires at all.
|
||||
///
|
||||
/// Registering a second, inert dispatcher closes both halves of that race:
|
||||
///
|
||||
/// * constructing it rebuilds every *already-registered* callsite's interest
|
||||
/// against the live dispatcher set — which includes the caller's subscriber —
|
||||
/// repairing whatever a sibling may already have poisoned; and
|
||||
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
|
||||
/// a callsite registered *later* by any thread is resolved against that live
|
||||
/// set instead of the registering thread's `NoSubscriber`.
|
||||
///
|
||||
/// Call this **after** installing the test's subscriber, and hold the returned
|
||||
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
|
||||
/// nextest runs each test in its own process, where there is no sibling thread
|
||||
/// to lose the race to (see `docs/testing/README.md`).
|
||||
#[must_use = "callsite interest is only pinned while the returned guard is held"]
|
||||
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
|
||||
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
|
||||
}
|
||||
@@ -356,6 +356,14 @@ impl FileMeta {
|
||||
.versions
|
||||
.partition_point(|existing| existing.header.sorts_before(&new_shallow.header));
|
||||
self.versions.insert(insert_pos, new_shallow);
|
||||
// `partition_point` only returns the canonical slot when `versions` is
|
||||
// already canonically ordered, and nothing establishes that: `FileMeta::load`
|
||||
// replays the on-disk order verbatim, and metadata written before the
|
||||
// canonical-order fix ordered equal-`mod_time` ties by insertion rather than
|
||||
// by `sorts_before`. Re-sort so an insert leaves the list canonical either
|
||||
// way, matching what `set_idx` already does on the replace path. Cheap: after
|
||||
// a correctly placed insert the slice is a single sorted run.
|
||||
self.sort_by_mod_time();
|
||||
Ok(())
|
||||
|
||||
// if !ver.valid() {
|
||||
@@ -1215,6 +1223,57 @@ mod test {
|
||||
assert!(fm.versions[0].header.sorts_before(&fm.versions[1].header));
|
||||
}
|
||||
|
||||
/// `add_version_filemata` positions an inserted version with
|
||||
/// `partition_point(sorts_before)`, which only yields the canonical slot when
|
||||
/// `versions` is already canonically ordered. Nothing establishes that
|
||||
/// precondition on load: `FileMeta::unmarshal_msg` pushes versions in file
|
||||
/// order, and metadata written before the canonical-order fix ordered
|
||||
/// equal-`mod_time` ties by insertion instead of by `sorts_before`. An insert
|
||||
/// into such a list must still leave it canonical, the same way `set_idx`
|
||||
/// already re-sorts on the replace path.
|
||||
#[test]
|
||||
fn add_version_filemata_canonicalizes_versions_loaded_out_of_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
let first = version_for_ordering(VersionType::Object, Uuid::from_u128(70), mod_time, 70);
|
||||
let second = version_for_ordering(VersionType::Object, Uuid::from_u128(80), mod_time, 80);
|
||||
let (early, late) = if first.header().sorts_before(&second.header()) {
|
||||
(first, second)
|
||||
} else {
|
||||
(second, first)
|
||||
};
|
||||
|
||||
// Persist the pair the wrong way round to emulate a pre-canonical-order
|
||||
// xl.meta, bypassing add_version_filemata so the bad order really lands on
|
||||
// the wire.
|
||||
let mut legacy = FileMeta::new();
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(late).expect("shallow later version"));
|
||||
legacy
|
||||
.versions
|
||||
.push(FileMetaShallowVersion::try_from(early).expect("shallow earlier version"));
|
||||
|
||||
let mut loaded =
|
||||
FileMeta::load(&legacy.marshal_msg().expect("serialize legacy metadata")).expect("reload legacy metadata");
|
||||
assert!(
|
||||
!loaded.versions[0].header.sorts_before(&loaded.versions[1].header),
|
||||
"fixture must reach add_version_filemata non-canonically ordered"
|
||||
);
|
||||
|
||||
loaded
|
||||
.add_version_filemata(version_for_ordering(VersionType::Delete, Uuid::from_u128(90), mod_time, 90))
|
||||
.expect("insert equal-time delete marker");
|
||||
|
||||
assert_eq!(loaded.versions.len(), 3);
|
||||
assert!(
|
||||
loaded
|
||||
.versions
|
||||
.windows(2)
|
||||
.all(|pair| pair[0].header.sorts_before(&pair[1].header)),
|
||||
"insert must leave the version list canonically ordered"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for backlog#799 B16: replication reset state must be
|
||||
/// persisted under both internal prefixes, never as a bare ARN. A bare-ARN
|
||||
/// key (produced by `ObjectInfo::replication_state`) has no internal prefix,
|
||||
|
||||
@@ -26,6 +26,9 @@ documentation = "https://docs.rs/rustfs-heal/latest/rustfs_heal/"
|
||||
keywords = ["RustFS", "heal", "erasure-coding", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
@@ -53,7 +56,7 @@ serial_test = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -482,7 +482,7 @@ impl HealChannelProcessor {
|
||||
request_id: client_token,
|
||||
success: false,
|
||||
data: None,
|
||||
error: Some(error_text.clone()),
|
||||
error: Some(error_text),
|
||||
};
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
|
||||
@@ -576,7 +576,7 @@ mod tests {
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
assert_eq!(handler.event_count(), 3);
|
||||
}
|
||||
|
||||
@@ -593,7 +593,7 @@ mod tests {
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone()); // Should remove oldest
|
||||
handler.add_event(event); // Should remove oldest
|
||||
|
||||
assert_eq!(handler.event_count(), 2);
|
||||
}
|
||||
@@ -610,7 +610,7 @@ mod tests {
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
|
||||
let events = handler.get_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
|
||||
@@ -2892,7 +2892,7 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
|
||||
gauge!(
|
||||
"rustfs_heal_task_running",
|
||||
"type" => type_label.to_string(),
|
||||
"set" => set_label.clone()
|
||||
"set" => set_label
|
||||
)
|
||||
.set(count as f64);
|
||||
}
|
||||
@@ -3482,7 +3482,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);
|
||||
|
||||
let mut running = HashMap::new();
|
||||
running.insert("pool_0_set_1".to_string(), 1);
|
||||
|
||||
@@ -28,6 +28,7 @@ use rustfs_heal::heal::storage::{
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
future::Future,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
@@ -52,15 +53,9 @@ fn versioned_test_data(seed: u8) -> Vec<u8> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Disable the dangling-delete grace window so the destructive path is genuinely
|
||||
/// LIVE in these tests: without the decision-1 guard, a recoverable version WOULD
|
||||
/// be dangling-deleted here. With grace at its 1h default the delete path would be
|
||||
/// masked and the test would prove nothing.
|
||||
fn disable_dangling_grace() {
|
||||
// Safe under nextest: each test runs in its own process and is `#[serial]`.
|
||||
unsafe {
|
||||
std::env::set_var(GRACE_ENV, "0");
|
||||
}
|
||||
/// Disable the grace window while the destructive heal decision is evaluated.
|
||||
async fn with_dangling_grace_disabled<T>(future: impl Future<Output = T>) -> T {
|
||||
temp_env::async_with_vars([(GRACE_ENV, Some("0"))], future).await
|
||||
}
|
||||
|
||||
/// Build a real N-disk single-set `ECStore` + `ECStoreHealStorage` via the
|
||||
@@ -250,7 +245,6 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn union_meta_lost_data_present_is_repaired_not_destroyed() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-meta-lost";
|
||||
let object = "obj.bin";
|
||||
@@ -268,10 +262,10 @@ mod serial_tests {
|
||||
}
|
||||
|
||||
// Heal the version through the real heal storage (Deep).
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(
|
||||
error.is_none(),
|
||||
"recoverable version must heal without error (must NOT be dangling-deleted): {error:?}"
|
||||
@@ -300,7 +294,6 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_torn_minority_is_dangling_deleted_with_grace_zero() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-torn";
|
||||
let object = "obj.bin";
|
||||
@@ -318,10 +311,10 @@ mod serial_tests {
|
||||
// is a real destructive action, not a no-op).
|
||||
assert_eq!(count_part_files(&object_dir(&disk_paths[0], bucket, object)), 1);
|
||||
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
// A dangling delete reports the version as gone (FileVersionNotFound),
|
||||
// proving the destructive path fired for a genuinely torn write.
|
||||
assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
|
||||
@@ -349,7 +342,6 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_restores_subquorum_but_reconstructable_version_wider_set() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(8).await;
|
||||
let bucket = "b920-reconstruct";
|
||||
let object = "obj.bin";
|
||||
@@ -373,10 +365,10 @@ mod serial_tests {
|
||||
"disk-walk must enumerate the reconstructable sub-quorum version"
|
||||
);
|
||||
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "reconstructable version must heal cleanly: {error:?}");
|
||||
|
||||
// part.* + xl.meta physically restored on the 4 wiped disks.
|
||||
@@ -463,7 +455,6 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn offline_disk_during_walk_does_not_dangling_delete() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b920-offline";
|
||||
let object = "obj.bin";
|
||||
@@ -491,8 +482,7 @@ mod serial_tests {
|
||||
remove: false,
|
||||
..Default::default()
|
||||
};
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &normal_opts)
|
||||
let (_result, error) = with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &normal_opts))
|
||||
.await
|
||||
.expect("heal_object call must not itself error");
|
||||
assert!(error.is_none(), "quorum-present object must not be destroyed: {error:?}");
|
||||
@@ -510,7 +500,6 @@ mod serial_tests {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn deep_heal_keeps_present_ec2_plus_2_shards_healthy() {
|
||||
disable_dangling_grace();
|
||||
let (disk_paths, ecstore, heal_storage) = heal_env_n(4).await;
|
||||
let bucket = "b1044-deep-verify";
|
||||
let object = "obj.bin";
|
||||
@@ -524,10 +513,10 @@ mod serial_tests {
|
||||
assert_eq!(count_part_files(&object_dir(disk, bucket, object)), 1, "intact shard must remain present");
|
||||
}
|
||||
|
||||
let (_result, error) = heal_storage
|
||||
.heal_object(bucket, object, Some(&v1), &deep_heal_opts())
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
let (_result, error) =
|
||||
with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
|
||||
.await
|
||||
.expect("deep heal_object call must not itself error");
|
||||
assert!(error.is_none(), "Deep heal must retain the three intact EC2+2 shards: {error:?}");
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -390,8 +390,10 @@ mod serial_tests {
|
||||
|
||||
// ─── 1️⃣ delete format.json on one disk ──────────────
|
||||
let format_path = disk_paths[0].join(".rustfs.sys").join("format.json");
|
||||
std::fs::remove_dir_all(&disk_paths[0]).expect("failed to delete all contents under disk_paths[0]");
|
||||
let failed_disk_path = disk_paths[0].with_extension("failed");
|
||||
std::fs::rename(&disk_paths[0], &failed_disk_path).expect("failed to detach disk_paths[0]");
|
||||
std::fs::create_dir_all(&disk_paths[0]).expect("failed to recreate disk_paths[0] directory");
|
||||
assert!(!format_path.exists(), "replacement disk already contains format.json before heal");
|
||||
println!("✅ Deleted format.json on disk: {:?}", disk_paths[0]);
|
||||
|
||||
let (_format_result, format_error) = heal_storage.heal_format(false).await.expect("failed to run heal_format");
|
||||
|
||||
@@ -556,7 +556,7 @@ where
|
||||
Ok(now)
|
||||
}
|
||||
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
let mut m = HashMap::new();
|
||||
|
||||
self.api.load_policy_docs(&mut m).await?;
|
||||
@@ -588,6 +588,15 @@ where
|
||||
Ok(filtered)
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use list_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.list_policies(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn merge_policies(&self, name: &str) -> (String, Policy) {
|
||||
let mut policies = Vec::new();
|
||||
let mut to_merge = Vec::new();
|
||||
@@ -2184,7 +2193,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
|
||||
pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
|
||||
let default_policies = &DEFAULT_POLICIES;
|
||||
default_policies
|
||||
.iter()
|
||||
@@ -2201,6 +2210,15 @@ pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use get_default_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
|
||||
get_default_policies()
|
||||
}
|
||||
|
||||
fn set_default_canned_policies(policies: &mut HashMap<String, PolicyDoc>) {
|
||||
let default_policies = &DEFAULT_POLICIES;
|
||||
for (k, v) in default_policies.iter() {
|
||||
@@ -2900,7 +2918,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_get_default_policies() {
|
||||
let policies = get_default_policyes();
|
||||
let policies = get_default_policies();
|
||||
|
||||
// Should contain some default policies
|
||||
assert!(!policies.is_empty());
|
||||
@@ -2913,6 +2931,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_get_default_policyes_matches_current_api() {
|
||||
assert_eq!(get_default_policyes().len(), get_default_policies().len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_token_signing_key() {
|
||||
// This function returns the global action credential's secret key
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::{
|
||||
cache::{Cache, CacheEntity},
|
||||
error::{is_err_no_such_policy, is_err_no_such_user},
|
||||
keyring,
|
||||
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes},
|
||||
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policies},
|
||||
root_credentials,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
@@ -1127,7 +1127,7 @@ impl Store for ObjectStore {
|
||||
let cache_snapshot = cache.snapshot();
|
||||
let listed_config_items = self.list_all_iamconfig_items().await?;
|
||||
|
||||
let mut policy_docs_cache = CacheEntity::new(get_default_policyes());
|
||||
let mut policy_docs_cache = CacheEntity::new(get_default_policies());
|
||||
|
||||
if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) {
|
||||
// Load in fixed-size chunks so each policy is fetched exactly once.
|
||||
|
||||
+20
-5
@@ -18,7 +18,7 @@ use crate::error::is_err_no_such_temp_account;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::federation::OIDC_VIRTUAL_PARENT_CLAIM;
|
||||
use crate::manager::extract_jwt_claims;
|
||||
use crate::manager::get_default_policyes;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::manager::{IamCache, IamSyncMetricsSnapshot};
|
||||
use crate::store::GroupInfo;
|
||||
use crate::store::MappedPolicy;
|
||||
@@ -249,7 +249,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn delete_policy(&self, name: &str, notify: bool) -> Result<()> {
|
||||
for k in get_default_policyes().keys() {
|
||||
for k in get_default_policies().keys() {
|
||||
if k == name {
|
||||
return Err(Error::other("system policy can not be deleted"));
|
||||
}
|
||||
@@ -291,8 +291,17 @@ impl<T: Store> IamSys<T> {
|
||||
self.store.api.load_mapped_policies(user_type, is_group, m).await
|
||||
}
|
||||
|
||||
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.store.list_policies(bucket_name).await
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use list_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.store.list_polices(bucket_name).await
|
||||
self.list_policies(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn list_policy_docs(&self, bucket_name: &str) -> Result<HashMap<String, PolicyDoc>> {
|
||||
@@ -1683,11 +1692,17 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::cache::{Cache, CacheEntity};
|
||||
use crate::error::Error;
|
||||
use crate::manager::get_default_policyes;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
|
||||
use rustfs_policy::policy::Args;
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_list_polices_api_is_available() {
|
||||
let _ = IamSys::<StsTestMockStore>::list_polices;
|
||||
}
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
|
||||
use serde_json::Value;
|
||||
@@ -1925,7 +1940,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn load_all(&self, cache: &Cache) -> Result<()> {
|
||||
let mut policy_docs = get_default_policyes();
|
||||
let mut policy_docs = get_default_policies();
|
||||
let custom_claim_policy =
|
||||
Policy::parse_config(CUSTOM_STS_CLAIM_POLICY_JSON.as_bytes()).expect("custom STS claim policy should parse");
|
||||
policy_docs.insert(CUSTOM_STS_CLAIM_POLICY.to_string(), PolicyDoc::new(custom_claim_policy));
|
||||
|
||||
@@ -67,6 +67,7 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
@@ -159,6 +160,7 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_scope_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
}
|
||||
|
||||
@@ -180,6 +182,7 @@ pub struct InternodeMetrics {
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_scope_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -431,6 +434,13 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
|
||||
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
|
||||
pub fn record_replay_scope_fallback(&self) {
|
||||
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||
@@ -488,6 +498,7 @@ impl InternodeMetrics {
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
@@ -510,6 +521,7 @@ impl InternodeMetrics {
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -887,6 +899,19 @@ mod tests {
|
||||
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
|
||||
metrics.record_replay_scope_fallback();
|
||||
metrics.record_replay_scope_fallback();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
|
||||
|
||||
metrics.reset_for_test();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
|
||||
// Unique addr keeps this independent of the process-global registry / other tests.
|
||||
|
||||
@@ -25,6 +25,9 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
|
||||
categories = ["authentication", "web-programming"]
|
||||
authors.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
|
||||
@@ -44,7 +44,9 @@ argon2 = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
base64 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
subtle = { workspace = true }
|
||||
zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
@@ -55,7 +57,7 @@ tempfile = { workspace = true }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
# Additional dependencies
|
||||
md5 = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
|
||||
+209
-126
@@ -35,7 +35,6 @@ use std::collections::HashMap;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Reject key identifiers that would not name a single file directly inside the key
|
||||
@@ -77,8 +76,6 @@ const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
|
||||
/// Local KMS client that stores keys in local files
|
||||
pub struct LocalKmsClient {
|
||||
config: LocalConfig,
|
||||
/// In-memory cache of loaded keys for performance
|
||||
key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
|
||||
/// Master encryption key for encrypting stored keys
|
||||
master_cipher: Option<Aes256Gcm>,
|
||||
/// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
|
||||
@@ -139,13 +136,14 @@ impl LocalKmsClient {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
let client = Self {
|
||||
config,
|
||||
key_cache: RwLock::new(HashMap::new()),
|
||||
master_cipher,
|
||||
legacy_master_cipher,
|
||||
dek_crypto: AesDekCrypto::new(),
|
||||
})
|
||||
};
|
||||
client.validate_existing_keys().await?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Derive a 256-bit key from the master key string using a persistent Argon2id salt.
|
||||
@@ -193,10 +191,35 @@ impl LocalKmsClient {
|
||||
|
||||
let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN];
|
||||
rand::rng().fill(&mut salt[..]);
|
||||
fs::write(&salt_path, salt).await?;
|
||||
Self::set_file_permissions(&salt_path, config.file_permissions).await?;
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
let temp_path = config
|
||||
.key_dir
|
||||
.join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, salt).await?;
|
||||
Self::set_file_permissions(&temp_path, config.file_permissions).await?;
|
||||
match fs::hard_link(&temp_path, &salt_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file");
|
||||
}
|
||||
debug!(path = ?salt_path, "Local KMS master key salt created");
|
||||
Ok(salt)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
let bytes = fs::read(&salt_path).await?;
|
||||
bytes.try_into().map_err(|_| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} must be exactly {} bytes",
|
||||
salt_path.display(),
|
||||
LOCAL_KMS_MASTER_KEY_SALT_LEN
|
||||
))
|
||||
})
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -242,6 +265,12 @@ impl LocalKmsClient {
|
||||
|
||||
let content = fs::read(&key_path).await?;
|
||||
let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
|
||||
if stored_key.key_id != key_id {
|
||||
return Err(KmsError::invalid_key(format!(
|
||||
"Local KMS key file identity mismatch: expected {key_id:?}, found {:?}",
|
||||
stored_key.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
let encrypted_bytes = BASE64
|
||||
.decode(&stored_key.encrypted_key_material)
|
||||
@@ -326,8 +355,43 @@ impl LocalKmsClient {
|
||||
/// Save a master key to disk
|
||||
async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
// Encrypt key material if master cipher is available
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
|
||||
let key_path = self.master_key_path(&master_key.key_id)?;
|
||||
let content = self.encode_master_key(master_key, key_material)?;
|
||||
let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
match fs::hard_link(&temp_path, &key_path).await {
|
||||
Ok(()) => {
|
||||
if let Err(error) = fs::remove_file(&temp_path).await {
|
||||
warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file");
|
||||
}
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created");
|
||||
Ok(())
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(KmsError::key_already_exists(&master_key.key_id))
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
Err(error.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<Vec<u8>> {
|
||||
let (encrypted_key_material, nonce, at_rest_protection) = if let Some(ref cipher) = self.master_cipher {
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
rand::rng().fill(&mut nonce_bytes[..]);
|
||||
@@ -362,17 +426,7 @@ impl LocalKmsClient {
|
||||
at_rest_protection,
|
||||
};
|
||||
|
||||
let content = serde_json::to_vec_pretty(&stored_key)?;
|
||||
|
||||
// Write to temporary file first, then rename for atomicity
|
||||
let temp_path = key_path.with_extension("tmp");
|
||||
fs::write(&temp_path, &content).await?;
|
||||
Self::set_file_permissions(&temp_path, self.config.file_permissions).await?;
|
||||
|
||||
fs::rename(&temp_path, &key_path).await?;
|
||||
|
||||
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
|
||||
Ok(())
|
||||
serde_json::to_vec_pretty(&stored_key).map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Get the actual key material for a master key
|
||||
@@ -381,6 +435,23 @@ impl LocalKmsClient {
|
||||
Ok(key_material)
|
||||
}
|
||||
|
||||
async fn validate_existing_keys(&self) -> Result<()> {
|
||||
let mut entries = fs::read_dir(&self.config.key_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
if path.extension().is_none_or(|extension| extension != "key") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let key_id = path
|
||||
.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?;
|
||||
self.decode_stored_key(key_id).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encrypt data using a master key
|
||||
async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
|
||||
// Load the actual master key material
|
||||
@@ -513,11 +584,7 @@ impl KmsClient for LocalKmsClient {
|
||||
let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
|
||||
|
||||
// Save to disk
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Cache the key
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
self.save_new_master_key(&master_key, &key_material).await?;
|
||||
|
||||
debug!(key_id, "Local KMS master key created");
|
||||
Ok(master_key)
|
||||
@@ -526,23 +593,7 @@ impl KmsClient for LocalKmsClient {
|
||||
async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
|
||||
debug!("Describing key: {}", key_id);
|
||||
|
||||
// Check cache first
|
||||
{
|
||||
let cache = self.key_cache.read().await;
|
||||
if let Some(master_key) = cache.get(key_id) {
|
||||
return Ok(master_key.clone().into());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let master_key = self.load_master_key(key_id).await?;
|
||||
|
||||
// Update cache
|
||||
{
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
}
|
||||
|
||||
Ok(master_key.into())
|
||||
}
|
||||
|
||||
@@ -602,10 +653,6 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key enabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -621,10 +668,6 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key disabled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -646,10 +689,6 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion scheduled");
|
||||
Ok(())
|
||||
}
|
||||
@@ -665,31 +704,17 @@ impl KmsClient for LocalKmsClient {
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key);
|
||||
|
||||
debug!(key_id, "Local KMS key deletion canceled");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
|
||||
debug!("Rotating key: {}", key_id);
|
||||
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
master_key.version += 1;
|
||||
master_key.rotated_at = Some(Zoned::now());
|
||||
|
||||
// Generate new key material
|
||||
let key_material = generate_key_material(&master_key.algorithm)?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
debug!(key_id, "Local KMS key rotated");
|
||||
Ok(master_key)
|
||||
if !fs::try_exists(self.master_key_path(key_id)?).await? {
|
||||
return Err(KmsError::key_not_found(key_id));
|
||||
}
|
||||
Err(KmsError::invalid_operation(
|
||||
"Local KMS key rotation is unavailable until historical key versions can be retained",
|
||||
))
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<()> {
|
||||
@@ -745,11 +770,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
let key_id = request.key_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
|
||||
|
||||
// `save_master_key` writes through a temp file and renames over the destination, so
|
||||
// creating a key under an existing name would replace its material and silently
|
||||
// destroy the ability to decrypt everything wrapped under it. The sibling
|
||||
// `KmsClient::create_key` has always refused this; the backend path did not, and
|
||||
// this is the path the admin API uses.
|
||||
if self.client.master_key_path(&key_id)?.exists() {
|
||||
return Err(KmsError::key_already_exists(&key_id));
|
||||
}
|
||||
@@ -767,11 +787,8 @@ impl KmsBackend for LocalKmsBackend {
|
||||
request.description.clone(),
|
||||
);
|
||||
|
||||
// Save to disk and cache
|
||||
self.client.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.clone(), master_key.clone());
|
||||
// Save to disk
|
||||
self.client.save_new_master_key(&master_key, &key_material).await?;
|
||||
|
||||
master_key
|
||||
};
|
||||
@@ -888,10 +905,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.await
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
|
||||
// Remove from cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.remove(key_id);
|
||||
|
||||
debug!(key_id, "Local KMS key deleted immediately");
|
||||
|
||||
// Return success response for immediate deletion
|
||||
@@ -935,10 +948,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -986,10 +995,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
|
||||
self.client.save_master_key(&master_key, &existing_key_material).await?;
|
||||
|
||||
// Update cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
cache.insert(key_id.to_string(), master_key.clone());
|
||||
|
||||
// Convert master_key to KeyMetadata for response
|
||||
let key_metadata = KeyMetadata {
|
||||
key_id: master_key.key_id.clone(),
|
||||
@@ -1189,6 +1194,18 @@ mod tests {
|
||||
.expect("stored encrypted key should deserialize");
|
||||
assert_eq!(stored.at_rest_protection, StoredKeyProtection::EncryptedMasterKey);
|
||||
assert_eq!(stored.nonce.len(), 12);
|
||||
|
||||
let wrong_master_error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: client.config.key_dir.clone(),
|
||||
master_key: Some("wrong-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("wrong master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1228,17 +1245,57 @@ mod tests {
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let client_without_master = LocalKmsClient::new(config)
|
||||
.await
|
||||
.expect("client without master key should still initialize in dev-mode tests");
|
||||
|
||||
let err = client_without_master
|
||||
.describe_key("encrypted-key", None)
|
||||
.await
|
||||
.expect_err("encrypted key should require a master key to read");
|
||||
let err = match LocalKmsClient::new(config).await {
|
||||
Ok(_) => panic!("initialization must reject an unreadable encrypted key"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(err.to_string().contains("requires a configured master key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_key_rotation_is_rejected_without_overwriting_key_material() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
let key_id = "rotation-key";
|
||||
client.create_key(key_id, "AES_256", None).await.expect("create key");
|
||||
let original_material = client.get_key_material(key_id).await.expect("load original material");
|
||||
|
||||
let error = client
|
||||
.rotate_key(key_id, None)
|
||||
.await
|
||||
.expect_err("rotation must remain unavailable without historical key versions");
|
||||
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }));
|
||||
assert_eq!(
|
||||
client.get_key_material(key_id).await.expect("reload original material"),
|
||||
original_material
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_rejects_key_file_with_mismatched_embedded_id() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("file-name", "AES_256", None).await.expect("create key");
|
||||
let key_path = client.master_key_path("file-name").expect("valid key id");
|
||||
let mut stored: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode key file");
|
||||
stored["key_id"] = serde_json::json!("embedded-name");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&stored).expect("encode mismatched key"))
|
||||
.await
|
||||
.expect("write mismatched key");
|
||||
|
||||
let error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: None,
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("mismatched key identity must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(error, KmsError::InvalidKey { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
@@ -1329,17 +1386,6 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("write beta.5 fixture");
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
|
||||
let client = LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("beta5-test-master-key".to_string()),
|
||||
@@ -1353,24 +1399,34 @@ mod tests {
|
||||
.await
|
||||
.expect("decrypt beta.5 SHA-256 protected key");
|
||||
assert_eq!(material, vec![0x42; 32]);
|
||||
|
||||
let mut explicit_protection = stored_key.clone();
|
||||
let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
|
||||
explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
|
||||
explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
|
||||
fs::write(
|
||||
temp_dir.path().join("beta5-explicit-key.key"),
|
||||
serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
|
||||
)
|
||||
.await
|
||||
.expect("write explicit-protection fixture");
|
||||
let explicit_error = client
|
||||
.get_key_material("beta5-explicit-key")
|
||||
.await
|
||||
.expect_err("explicit current protection must not fall back to the beta.5 KDF");
|
||||
assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
|
||||
|
||||
let wrong_key_client = LocalKmsClient::new(LocalConfig {
|
||||
let wrong_key_error = match LocalKmsClient::new(LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("wrong-beta5-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
})
|
||||
.await
|
||||
.expect("initialize local KMS with wrong beta.5 master key");
|
||||
let error = wrong_key_client
|
||||
.get_key_material("beta5-key")
|
||||
.await
|
||||
.expect_err("wrong beta.5 master key must not decrypt the fixture");
|
||||
assert!(matches!(error, KmsError::CryptographicError { .. }));
|
||||
{
|
||||
Ok(_) => panic!("wrong beta.5 master key must fail initialization"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. }));
|
||||
}
|
||||
|
||||
/// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path
|
||||
@@ -1429,9 +1485,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// R07-CAN-103: `save_master_key` writes a temp file and renames over the destination,
|
||||
/// so creating a key under an existing name would replace its material and silently
|
||||
/// destroy the ability to decrypt anything wrapped under it.
|
||||
/// R07-CAN-103: creating a duplicate key must preserve its original material.
|
||||
#[tokio::test]
|
||||
async fn backend_create_key_refuses_to_replace_existing_key_material() {
|
||||
let temp_dir = TempDir::new().expect("Failed to create temp dir");
|
||||
@@ -1469,4 +1523,33 @@ mod tests {
|
||||
.expect("original key material must survive the refused create");
|
||||
assert_eq!(original, after, "existing key material must not be replaced");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_backend_create_allows_only_one_writer() {
|
||||
let temp_dir = TempDir::new().expect("create key directory");
|
||||
let config = || LocalConfig {
|
||||
key_dir: temp_dir.path().to_path_buf(),
|
||||
master_key: Some("test-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
};
|
||||
let first = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create first client"),
|
||||
};
|
||||
let second = LocalKmsBackend {
|
||||
client: LocalKmsClient::new(config()).await.expect("create second client"),
|
||||
};
|
||||
let request = || CreateKeyRequest {
|
||||
key_name: Some("concurrent-key".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (first_result, second_result) = tokio::join!(first.create_key(request()), second.create_key(request()));
|
||||
assert_ne!(first_result.is_ok(), second_result.is_ok(), "exactly one create must succeed");
|
||||
let error = first_result
|
||||
.err()
|
||||
.or_else(|| second_result.err())
|
||||
.expect("one create must fail");
|
||||
assert!(matches!(error, KmsError::KeyAlreadyExists { .. }));
|
||||
assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,8 +177,8 @@ mod tests {
|
||||
let service1 = manager.get_encryption_service().await.expect("Service should be available");
|
||||
|
||||
// Reconfigure to new service (zero-downtime)
|
||||
let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
|
||||
let config2 = KmsConfig::local(temp_dir2.path().to_path_buf()).with_insecure_development_defaults();
|
||||
let mut config2 = config1;
|
||||
config2.timeout = std::time::Duration::from_secs(45);
|
||||
manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
|
||||
|
||||
// Verify version 2
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
@@ -27,6 +28,12 @@ use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::debug;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Data key for object encryption
|
||||
/// SECURITY: This struct automatically zeros sensitive key material when dropped
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -486,8 +493,7 @@ impl ObjectEncryptionService {
|
||||
|
||||
// Validate key MD5 if provided
|
||||
if let Some(expected_md5) = customer_key_md5 {
|
||||
let actual_md5 = md5::compute(customer_key);
|
||||
let actual_md5_hex = format!("{actual_md5:x}");
|
||||
let actual_md5_hex = md5_hex(customer_key);
|
||||
if actual_md5_hex != expected_md5.to_lowercase() {
|
||||
return Err(KmsError::validation_error("Customer key MD5 mismatch"));
|
||||
}
|
||||
|
||||
@@ -20,10 +20,12 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use arc_swap::ArcSwap;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -31,6 +33,53 @@ const LOG_COMPONENT_KMS: &str = "kms";
|
||||
const LOG_SUBSYSTEM_SERVICE: &str = "service";
|
||||
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
|
||||
|
||||
fn local_master_key_fingerprint(master_key: Option<&str>) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update([u8::from(master_key.is_some())]);
|
||||
if let Some(master_key) = master_key {
|
||||
digest.update(master_key.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
fn validate_local_transition(current: Option<&KmsConfig>, new: &KmsConfig) -> Result<()> {
|
||||
let Some(current) = current else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(current_local) = ¤t.backend_config else {
|
||||
return Ok(());
|
||||
};
|
||||
let BackendConfig::Local(new_local) = &new.backend_config else {
|
||||
return Err(KmsError::configuration_error("Local KMS backend cannot be changed after configuration"));
|
||||
};
|
||||
|
||||
if current_local.key_dir != new_local.key_dir {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS key directory cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current_local.file_permissions != new_local.file_permissions {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS file permissions cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
if current.allow_insecure_dev_defaults != new.allow_insecure_dev_defaults {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS development mode cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
let current_master_key = local_master_key_fingerprint(current_local.master_key.as_deref());
|
||||
let new_master_key = local_master_key_fingerprint(new_local.master_key.as_deref());
|
||||
if !bool::from(current_master_key.ct_eq(&new_master_key)) {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Local KMS master key cannot be changed after configuration",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// KMS service status
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum KmsServiceStatus {
|
||||
@@ -106,7 +155,12 @@ impl KmsServiceManager {
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
new_config.validate()?;
|
||||
{
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
}
|
||||
|
||||
// Update configuration
|
||||
{
|
||||
@@ -254,11 +308,9 @@ impl KmsServiceManager {
|
||||
"KMS service reconfiguring"
|
||||
);
|
||||
new_config.validate()?;
|
||||
|
||||
// Configure with new config
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config.clone());
|
||||
let config = self.config.read().await;
|
||||
validate_local_transition(config.as_ref(), &new_config)?;
|
||||
}
|
||||
|
||||
// Create new service version without stopping old one
|
||||
@@ -268,6 +320,11 @@ impl KmsServiceManager {
|
||||
// Get old version for logging (lock-free read)
|
||||
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
|
||||
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config);
|
||||
}
|
||||
|
||||
// Atomically switch to new service version (lock-free, instant CAS operation)
|
||||
// This is a true atomic operation - no waiting for locks, instant switch
|
||||
// Old service will be dropped when no more Arc references exist
|
||||
@@ -304,8 +361,6 @@ impl KmsServiceManager {
|
||||
Err(e) => {
|
||||
let err_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.clone());
|
||||
Err(KmsError::backend_error(&err_msg))
|
||||
}
|
||||
}
|
||||
@@ -477,4 +532,105 @@ mod tests {
|
||||
};
|
||||
assert!(static_config.secret_key.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
|
||||
use crate::types::{CreateKeyRequest, KeyUsage};
|
||||
use std::collections::HashMap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let config = |master_key: &str| {
|
||||
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
let BackendConfig::Local(local) = &mut config.backend_config else {
|
||||
panic!("local constructor must create local backend config");
|
||||
};
|
||||
local.master_key = Some(master_key.to_string());
|
||||
config.allow_insecure_dev_defaults = true;
|
||||
config
|
||||
};
|
||||
let manager = KmsServiceManager::new();
|
||||
manager
|
||||
.configure(config("working-master-key"))
|
||||
.await
|
||||
.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
manager
|
||||
.get_manager()
|
||||
.await
|
||||
.expect("running KMS manager")
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("existing-key".to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
policy: None,
|
||||
tags: HashMap::new(),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("create encrypted key");
|
||||
let service_version = manager.get_service_version().await;
|
||||
|
||||
let error = manager
|
||||
.reconfigure(config("wrong-master-key"))
|
||||
.await
|
||||
.expect_err("local master key change must be rejected");
|
||||
|
||||
assert!(error.to_string().contains("master key cannot be changed"));
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, service_version);
|
||||
let current = manager.get_config().await.expect("working config must remain");
|
||||
let BackendConfig::Local(local) = current.backend_config else {
|
||||
panic!("working config must remain local");
|
||||
};
|
||||
assert_eq!(local.master_key.as_deref(), Some("working-master-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_cannot_replace_existing_local_backend() {
|
||||
use base64::Engine as _;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut local = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
local.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(local.clone()).await.expect("configure local KMS");
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let error = manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
.expect_err("existing local backend must be immutable");
|
||||
|
||||
assert!(error.to_string().contains("backend cannot be changed"));
|
||||
let current = manager.get_config().await.expect("local config must remain");
|
||||
assert!(matches!(current.backend_config, BackendConfig::Local(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_allows_safe_local_runtime_settings_only() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut initial = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
initial.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(initial.clone()).await.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
|
||||
let mut updated = initial;
|
||||
updated.default_key_id = Some("evaluation-key".to_string());
|
||||
updated.timeout = std::time::Duration::from_secs(45);
|
||||
updated.enable_cache = false;
|
||||
manager
|
||||
.reconfigure(updated.clone())
|
||||
.await
|
||||
.expect("update safe local settings");
|
||||
|
||||
let current = manager.get_config().await.expect("updated config");
|
||||
assert_eq!(current.default_key_id, updated.default_key_id);
|
||||
assert_eq!(current.timeout, updated.timeout);
|
||||
assert!(!current.enable_cache);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user