gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-10-26 22:07:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(gateway): merge main into integration

Browse Source 
Source: main 0f24511535
Target: gateway/integration 8cf6187ff4

Resolve the e2e-full selection digest conflict by recomputing both
platform digests over the merged nextest listing (685 darwin / 693
linux): main adds two rpc_fault_proxy unit tests, integration keeps its
dual-stack differential and anonymous-access additions. Raise the rustfs
test-count baseline row to the merged count (5932).
This commit is contained in:
overtrue
2026-10-25 09:58:02 +08:00
parent 8cf6187ff4 0f24511535
commit 88b626e7a0
55 changed files with 4372 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/e2e-full-selection.txt
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=db39d25da89d9584f2f73bd27271022a93b77613a80e69ea85a6f5cc1c8b1f39
sha256-linux=61e8fe9426f4e6a1a578e684079f0bb81425eff4f851c333c4158f2ff79518f5
sha256-darwin=7893834728e2dadb98110c2aa018e6aa150c882c8a39c753e8095c3aef94cf0e
sha256-linux=fb95ac8ba4b68aec9182fe32a59923e6b0d37b45a62ce87ea7db0dc3c40375e6
.config/test-count-baseline.txt
+6 -6
View File
@@ -9,8 +9,8 @@
# the PR that adds tests; lowering one is a reviewed statement that tests were
# intentionally removed. Regenerate with:
# scripts/check_test_count_ratchet.sh --update
e2e_test 954 20
rustfs 5930 64
e2e_test 956 20
rustfs 5932 64
rustfs-audit 70 0
rustfs-checksums 26 0
rustfs-common 36 0
@@ -20,10 +20,10 @@ rustfs-connect-symbol-catalog 6 0
rustfs-credentials 24 0
rustfs-crypto 67 0
rustfs-data-usage 55 0
rustfs-ecstore 7060 1
rustfs-ecstore 7081 1
rustfs-extension-schema 23 0
rustfs-filemeta 355 0
rustfs-heal 976 3
rustfs-filemeta 364 0
rustfs-heal 984 3
rustfs-heal-contracts 7 0
rustfs-iam 394 0
rustfs-io-core 49 0
@@ -43,7 +43,7 @@ rustfs-policy 725 0
rustfs-protocols 738 1
rustfs-protos 66 0
rustfs-replication 209 0
rustfs-rio 205 0
rustfs-rio 206 0
rustfs-rio-v2 34 6
rustfs-s3-client 69 0
rustfs-s3-ops 8 0
Cargo.lock
Generated
+53 -58
View File
@@ -255,7 +255,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -266,7 +266,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -1461,12 +1461,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6"
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.1"
@@ -1618,7 +1612,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1637,7 +1631,7 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1869,7 +1863,7 @@ dependencies = [
"maybe-owned",
"rustix",
"rustix-linux-procfs",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
"winx",
]
@@ -2073,7 +2067,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"inout 0.1.4",
]
@@ -2296,9 +2290,9 @@ dependencies = [
[[package]]
name = "const_format"
version = "0.2.36"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e"
checksum = "a3decb4d6965448659923544a634a0571ba44cede4e11ef15a5a317126840454"
dependencies = [
"const_format_proc_macros",
"konst",
@@ -2306,9 +2300,9 @@ dependencies = [
[[package]]
name = "const_format_proc_macros"
version = "0.2.34"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
checksum = "941b570269a642506f08cb5b688c04bfa6cfd41a26f8bcdc49db8fd00c0ab659"
dependencies = [
"proc-macro2",
"quote",
@@ -2548,9 +2542,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-bigint"
version = "0.7.6"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb08299a6046833a879ace115697f12fd2bd6c8d3e93c909ffc9e0738e7cae0"
checksum = "cc907c45bb7e9ef8fc3778defd0f3cd8c81288f1b4e748eba064948f20c47122"
dependencies = [
"cpubits",
"ctutils",
@@ -2565,11 +2559,11 @@ dependencies = [
[[package]]
name = "crypto-common"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"typenum",
]
@@ -3910,7 +3904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -3945,7 +3939,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4137,9 +4131,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
[[package]]
name = "elliptic-curve"
version = "0.14.1"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65"
checksum = "0ad4496221fcfc21015fd8c0e26fcfab00504832dc0604f52acdd7bad2a8dc81"
dependencies = [
"base16ct",
"crypto-bigint",
@@ -4258,7 +4252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4575,9 +4569,9 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.14.9"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -4589,7 +4583,7 @@ version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rustversion",
"typenum",
]
@@ -5241,9 +5235,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.29.0"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f137bde4c97275aef220f3a1566f39721fbd53086a444c8d841b5f345719dc"
checksum = "62f2351902fa259b758c7303505273393f6b08172ab5bdd5739a2d2bc6d554e8"
dependencies = [
"arc-swap",
"async-channel",
@@ -5276,15 +5270,15 @@ dependencies = [
[[package]]
name = "hotpath-drain"
version = "0.29.0"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87df2604170a3a0b512a837eba4d5d18671e54df8d450cc894af36d41291dba5"
checksum = "83dab2340134d9976d62f9f5172f9c6584c7f521b092f258e0e58e412e160b53"
[[package]]
name = "hotpath-macros"
version = "0.29.0"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25dcd4a80153dd66b34b8ae5d02572d71cc3ff55aff397578cd471d645a8a740"
checksum = "a04bbe5701b097f224a08187a78b75bb9d9b4bbaff6a279c2a50ba73edbb7aac"
dependencies = [
"proc-macro2",
"quote",
@@ -5613,7 +5607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding 0.3.3",
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -6264,9 +6258,9 @@ dependencies = [
[[package]]
name = "lzma-rust2"
version = "0.22.0"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb"
checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa"
dependencies = [
"sha2 0.11.1",
]
@@ -6871,7 +6865,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -7004,7 +6998,7 @@ name = "oauth2"
version = "5.1.0"
source = "git+https://github.com/ramosbugs/oauth2-rs?rev=89585f4a280afd7b285eaf1c9be0068874e03d22#89585f4a280afd7b285eaf1c9be0068874e03d22"
dependencies = [
"base64 0.21.7",
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -8589,7 +8583,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10177,18 +10171,18 @@ dependencies = [
[[package]]
name = "rustfs-mimalloc"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91c0ac4b4c6f8246bb843c985b47f7f8ea3e654e02d965a5d6c19fc006813584"
checksum = "8301a578772fc9823d68e083e42c3b414613d588a0eac594c1ca6bc0bbe0653f"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "265332babe807f64b800b1a47d6d39e6897f249c153b962a6b1050f073816342"
checksum = "6eb4082853043cb29556b4b776b05f28c3146706f43c53869dbf1ef262e00e91"
dependencies = [
"cc",
]
@@ -11028,7 +11022,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11111,7 +11105,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11872,9 +11866,9 @@ dependencies = [
[[package]]
name = "smallvec"
version = "1.16.3"
version = "1.16.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
dependencies = [
"serde",
]
@@ -11913,7 +11907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -12064,7 +12058,7 @@ dependencies = [
"cfg-if",
"libc",
"psm",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -12370,10 +12364,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -13046,9 +13040,9 @@ checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
[[package]]
name = "typed-path"
version = "0.12.4"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c"
checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e"
[[package]]
name = "typeid"
@@ -13467,7 +13461,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -14028,9 +14022,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.0.5"
version = "9.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
dependencies = [
"aes 0.9.3",
"bzip2",
@@ -14049,6 +14043,7 @@ dependencies = [
"sha1 0.11.0",
"time",
"typed-path",
"windows-sys 0.61.2",
"zeroize",
"zopfli",
"zstd 0.14.3",
Cargo.toml
+4 -4
View File
@@ -338,7 +338,7 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
smallvec = { version = "1.16.3" }
smallvec = { version = "1.16.4" }
compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.0" }
@@ -366,7 +366,7 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.0.5"
zip = "9.1.0"
zstd = "0.14.3"
# Observability and Metrics
@@ -394,9 +394,9 @@ russh-sftp = "3.0.1"
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.1" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.29.0", default-features = false }
hotpath = { version = "0.29.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
crates/e2e_test/src/distributed/internode_tls_test.rs
+58 -22
View File
@@ -4,12 +4,13 @@
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
use crate::tls_gen;
use aws_sdk_s3::{Client, config::retry::RetryConfig, primitives::ByteStream};
use futures::future::try_join_all;
use rand::{Rng, SeedableRng, rngs::StdRng};
use sha2::{Digest, Sha256};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
const MIB: u64 = 1024 * 1024;
const READ_TIMEOUT: Duration = Duration::from_secs(180);
@@ -85,8 +86,10 @@ async fn verified_get(
expected: &ExpectedRead,
concurrent: bool,
) -> TestResult<serde_json::Value> {
let started = Instant::now();
let mut received = 0u64;
let mut stage = "response headers";
tokio::time::timeout(READ_TIMEOUT, async {
let started = Instant::now();
let mut request = client.get_object().bucket(bucket).key(&object.key);
if let Some((start, end)) = expected.range {
request = request.range(format!("bytes={start}-{end}"));
@@ -98,24 +101,37 @@ async fn verified_get(
} else {
assert_eq!(output.content_range(), None);
}
let mut count = 0u64;
stage = "response body";
let mut digest = Sha256::new();
while let Some(chunk) = output.body.next().await {
let chunk = chunk?;
count += u64::try_from(chunk.len())?;
assert!(count <= expected.length, "node {node}: GET exceeded the requested length");
received += u64::try_from(chunk.len())?;
assert!(received <= expected.length, "node {node}: GET exceeded the requested length");
digest.update(&chunk);
}
let actual_sha256 = hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower);
assert_eq!(count, expected.length, "node {node}: truncated GET {}", object.key);
assert_eq!(received, expected.length, "node {node}: truncated GET {}", object.key);
assert_eq!(actual_sha256, expected.sha256, "node {node}: corrupt GET {}", object.key);
eprintln!(
"TLS acceptance: GET node={node}, key={}, range={:?}, concurrent={concurrent}, bytes={received}, elapsed={:?}",
object.key, expected.range, started.elapsed()
);
Ok(serde_json::json!({
"operation": "get", "node": node, "key": object.key,
"range": expected.range, "bytes": count, "sha256": actual_sha256,
"range": expected.range, "bytes": received, "sha256": actual_sha256,
"concurrent": concurrent, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0,
}))
})
.await?
.await
.map_err(|error| {
std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!(
"TLS GET timed out: node={node}, key={}, range={:?}, concurrent={concurrent}, stage={stage}, received={received}/{}, elapsed={:?}: {error}",
object.key, expected.range, expected.length, started.elapsed()
),
)
})?
}
/// This case always exercises the full acceptance sizes; do not substitute
@@ -157,9 +173,13 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
{
return Err("acceptance report paths must be unused".into());
}
let log_directory = report_path
.as_ref()
.map_or_else(|| directory.join("logs"), |path| path.with_extension("logs"));
let log_directory = report_path.as_ref().map_or_else(
|| {
std::env::var_os("RUSTFS_E2E_LOG_DIR")
.map_or_else(|| directory.join("logs"), |root| PathBuf::from(root).join(unique_bucket("internode-tls")))
},
|path| path.with_extension("logs"),
);
std::fs::create_dir(&log_directory)?;
dist.capture_logs_under(&log_directory)?;
dist.cluster.start().await?;
@@ -183,7 +203,8 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
let mut results = Vec::new();
for length in [32 * 1024, 100 * MIB, 1024 * MIB] {
let fixture_directory = directory.clone();
let object = tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??;
let object = Arc::new(tokio::task::spawn_blocking(move || fixture(&fixture_directory, length)).await??);
eprintln!("TLS acceptance: fixture key={}, bytes={length}", object.key);
let started = Instant::now();
tokio::time::timeout(
READ_TIMEOUT,
@@ -194,7 +215,18 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
.body(ByteStream::from_path(&object.path).await?)
.send(),
)
.await??;
.await
.map_err(|error| {
std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!(
"TLS PUT timed out: key={}, bytes={length}, elapsed={:?}: {error}",
object.key,
started.elapsed()
),
)
})??;
eprintln!("TLS acceptance: PUT key={}, bytes={length}, elapsed={:?}", object.key, started.elapsed());
results.push(
serde_json::json!({"operation": "put", "bytes": length, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0}),
);
@@ -216,15 +248,19 @@ async fn four_node_tls_large_and_range_gets_preserve_bytes() -> TestResult {
results.push(verified_get(client, node, &bucket, &object, expected, false).await?);
}
}
results.extend(
try_join_all(
clients
.iter()
.enumerate()
.map(|(node, client)| verified_get(client, node, &bucket, &object, &object.reads[0], true)),
)
.await?,
);
// Each reader owns a Tokio task and its cooperative budget. Polling
// every body and synchronous digest in one parent task can let an
// always-ready reader starve its siblings and their deadlines.
let mut readers = JoinSet::new();
for (node, client) in clients.iter().enumerate() {
let client = client.clone();
let bucket = bucket.clone();
let object = Arc::clone(&object);
readers.spawn(async move { verified_get(&client, node, &bucket, &object, &object.reads[0], true).await });
}
while let Some(result) = readers.join_next().await {
results.push(result??);
}
tokio::fs::remove_file(&object.path).await?;
}
for node in 0..clients.len() {
crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs
+189 -47
View File
@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, unique_bucket};
use super::harness::{
TestResult, assert_object_bytes, cluster_admin_ok, configure_node_logs, payload_for, retrying_put, sha256_hex, unique_bucket,
wait_until,
};
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
use aws_sdk_s3::{
@@ -84,12 +87,19 @@ fn create(
tokio::spawn(async move { request.send().await })
}
async fn wait_for_faults(pauses: &[RpcPause]) -> TestResult {
// Distributed locks use adaptive fan-out and may never contact an
// unnecessary voter. Every selected peer remains gated; require a real
// request hit without waiting for an unused peer to exhaust the S3 budget.
let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
result?;
async fn wait_for_faults(pauses: &[RpcPause], methods: &[&str]) -> TestResult {
if methods.iter().any(|method| matches!(*method, "Lock" | "LockBatch")) {
// Adaptive lock fan-out may not contact an unnecessary voter. All
// selected peers remain gated; require an actual scoped request hit.
let (result, _, _) = futures::future::select_all(pauses.iter().map(|pause| Box::pin(pause.wait_for_hits(1)))).await;
result?;
} else {
// Storage mutations fan out eagerly. Every selected peer must really
// receive this upload's RPC, so an unrelated failure cannot fake quorum.
for pause in pauses {
pause.wait_for_hits(1).await?;
}
}
Ok(())
}
@@ -130,6 +140,83 @@ async fn abort(client: &Client, bucket: &str, key: &str, upload: &str) -> TestRe
Ok(())
}
// Fault removal does not synchronously clear existing server leases or drive
// health state. Keep the injected operation single-attempt, then require
// bounded recovery of the exact cleanup API instead of assuming instant heal.
async fn recover_after_fault<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: std::fmt::Debug + ProvideErrorMetadata,
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
let mut last_failure = String::new();
let outcome = tokio::time::timeout(DEADLINE, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().and_then(|error| {
let response = error.raw_response()?;
if response.status().as_u16() != 503
|| error.as_service_error()?.meta().code() != Some("ServiceUnavailable")
|| !matches!(
response.headers().get("x-rustfs-error-cause"),
Some("namespace_lock_timeout" | "namespace_lock_unavailable" | "storage_write_quorum")
)
{
return None;
}
let delay = match response.headers().get("retry-after") {
Some("1") => Duration::from_secs(1),
None => Duration::from_millis(100),
Some(_) => return None,
};
last_failure = format!(
"cause={:?}, request_id={:?}",
response.headers().get("x-rustfs-error-cause"),
response.headers().get("x-amz-request-id")
);
Some(delay)
});
match retry {
Some(delay) => tokio::time::sleep(delay).await,
None => return result,
}
}
})
.await;
eprintln!(
"MPU recovery: operation={operation}, attempts={attempts}, elapsed={:?}, last={last_failure}",
started.elapsed()
);
outcome.map_err(|_| format!("{operation} did not recover within {DEADLINE:?}: {last_failure}").into())
}
async fn abort_after_fault(client: &Client, bucket: &str, key: &str, upload: &str, allow_missing: bool) -> TestResult {
match recover_after_fault(&format!("Abort {key}"), || {
client
.abort_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload)
.send()
})
.await?
{
Ok(_) => {}
Err(error) if allow_missing && error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {}
Err(error) => return Err(error.into()),
}
let missing = recover_after_fault(&format!("ListParts {key}"), || {
client.list_parts().bucket(bucket).key(key).upload_id(upload).send()
})
.await?
.expect_err("a recovered Abort must remove the upload");
assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
Ok(())
}
async fn round_trip(clients: &[Client], bucket: &str, key: &str, size: usize) -> TestResult {
let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
@@ -180,6 +267,31 @@ async fn assert_no_uploads(clients: &[Client], bucket: &str) -> TestResult {
Ok(())
}
async fn assert_no_uploads_after_fault(clients: &[Client], bucket: &str) -> TestResult {
for client in clients {
let uploads = recover_after_fault("ListMultipartUploads after fault", || {
client.list_multipart_uploads().bucket(bucket).send()
})
.await??;
assert!(uploads.uploads().is_empty(), "unexpected residual uploads: {:?}", uploads.uploads());
}
Ok(())
}
async fn assert_object_bytes_after_fault(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = recover_after_fault(&format!("GetObject {key} after fault"), || {
client.get_object().bucket(bucket).key(key).send()
})
.await??;
// A successful response must have a complete, correct body. Streaming
// failures are not retried or converted into a successful comparison.
let bytes = output.body.collect().await?.into_bytes();
if bytes.as_ref() != expected {
return Err(format!("object {key} bytes differ: expected {} bytes, got {}", expected.len(), bytes.len()).into());
}
Ok(())
}
// This fixture inspects RustFS RPCs, so use the server-generated upload's
// canonical suffix and fail explicitly if its encoding changes.
fn upload_scope(upload_id: &str) -> TestResult<String> {
@@ -191,6 +303,14 @@ fn upload_scope(upload_id: &str) -> TestResult<String> {
Ok(suffix.to_owned())
}
fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<String> {
Ok(format!(
"{}/{}",
sha256_hex(format!("{bucket}/{key}").as_bytes()),
upload_scope(upload_id)?
))
}
async fn with_paused_peers<T: Send + 'static>(
proxies: &[RpcFaultProxy],
methods: &[&'static str],
@@ -204,7 +324,7 @@ async fn with_paused_peers<T: Send + 'static>(
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
.collect::<Result<_, _>>()?;
let task = tokio::spawn(request);
wait_for_faults(&pauses).await?;
wait_for_faults(&pauses, methods).await?;
let result = tokio::time::timeout(DEADLINE, task).await??;
for pause in &pauses {
pause.discard_pending();
@@ -234,6 +354,29 @@ fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>
}
}
async fn wait_for_lock_peer_recovery(proxies: &[RpcFaultProxy], client: &Client, bucket: &str) -> TestResult {
let key = unique_bucket("peer-recovery");
let observers = proxies[1..]
.iter()
.map(|proxy| proxy.pause(&["Lock", "LockBatch"], usize::MAX, Some(&key)))
.collect::<Result<Vec<_>, _>>()?;
for observer in &observers {
observer.release();
}
wait_until(
DEADLINE,
|| async {
retrying_put(client, bucket, &key, vec![0x42; 1024], DEADLINE).await?;
// Quorum success alone can leave one peer's request breaker open.
// Two replies require a half-open peer to finish its first probe
// before admitting another RPC; every peer must rejoin the matrix.
Ok(observers.iter().all(|observer| observer.forwarded() >= 2))
},
"all lock peers recover after fault retirement",
)
.await
}
async fn phase_quorum_matrix(
proxies: &[RpcFaultProxy],
clients: &[Client],
@@ -274,13 +417,23 @@ async fn phase_quorum_matrix(
let request = clients[0].create_multipart_upload().bucket(bucket).key(&key);
let result = with_paused_peers(proxies, methods, &limits, bucket, async move { request.send().await }).await?;
if let Some(upload) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
abort(&clients[0], bucket, &key, upload.upload_id().ok_or("missing upload ID")?).await?;
abort_after_fault(&clients[0], bucket, &key, upload.upload_id().ok_or("missing upload ID")?, false).await?;
}
} else {
let upload = clients[0].create_multipart_upload().bucket(bucket).key(&key).send().await?;
let upload = clients[0]
.create_multipart_upload()
.bucket(bucket)
.key(&key)
.send()
.await
.map_err(|error| format!("setup Create failed for {key}: {error:?}"))?;
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
let scope = upload_scope(upload_id)?;
let body = payload_for(&key, 1024 * 1024);
let scope = if !locks && phase == "abort" {
upload_path_scope(bucket, &key, upload_id)?
} else {
upload_scope(upload_id)?
};
let body = payload_for(&key, if drives_per_node == 4 { 9 * 1024 * 1024 } else { 1024 * 1024 });
let initial = if phase != "part" {
Some(
clients[0]
@@ -308,7 +461,7 @@ async fn phase_quorum_matrix(
let result =
with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
if let Some(part) = quorum_outcome(result, unavailable == unavailable_at_quorum, locks)? {
clients[0]
let completion = clients[0]
.complete_multipart_upload()
.bucket(bucket)
.key(&key)
@@ -322,11 +475,11 @@ async fn phase_quorum_matrix(
.build(),
)
.build(),
)
.send()
.await?;
);
recover_after_fault(&format!("Complete {key} after part fault"), || completion.clone().send())
.await??;
for reader in clients {
assert_object_bytes(reader, bucket, &key, &body).await?;
assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
}
}
}
@@ -351,7 +504,7 @@ async fn phase_quorum_matrix(
with_paused_peers(proxies, methods, &limits, &scope, async move { request.send().await }).await?;
if quorum_outcome(result, unavailable == unavailable_at_quorum, locks)?.is_some() {
for reader in clients {
assert_object_bytes(reader, bucket, &key, &body).await?;
assert_object_bytes_after_fault(reader, bucket, &key, &body).await?;
}
}
}
@@ -367,36 +520,20 @@ async fn phase_quorum_matrix(
}
_ => unreachable!(),
}
// A failed completion can be ambiguous to an S3 client. Cleanup
// accepts an already completed/aborted session, never another error.
match clients[0]
.abort_multipart_upload()
.bucket(bucket)
.key(&key)
.upload_id(upload_id)
.send()
.await
{
Ok(_) => {}
Err(error) if error.as_service_error().and_then(|e| e.meta().code()) == Some("NoSuchUpload") => {}
Err(error) => return Err(error.into()),
}
let missing = clients[0]
.list_parts()
.bucket(bucket)
.key(&key)
.upload_id(upload_id)
.send()
.await
.expect_err("cleanup must remove the upload");
assert_eq!(missing.as_service_error().and_then(|e| e.meta().code()), Some("NoSuchUpload"));
clients[0].delete_object().bucket(bucket).key(&key).send().await?;
// Cleanup must recover even after a below-quorum mutation;
// only an already retired session may return NoSuchUpload.
abort_after_fault(&clients[0], bucket, &key, upload_id, true).await?;
recover_after_fault(&format!("DeleteObject {key} after {phase}"), || {
clients[0].delete_object().bucket(bucket).key(&key).send()
})
.await??;
}
assert_no_uploads(clients, bucket).await?;
wait_for_lock_peer_recovery(proxies, &clients[0], bucket).await?;
round_trip(clients, bucket, "recovered", 9 * 1024 * 1024).await?;
assert_no_uploads_after_fault(clients, bucket).await?;
}
}
round_trip(clients, bucket, "recovered", 9 * 1024 * 1024).await?;
assert_no_uploads(clients, bucket).await
Ok(())
}
#[tokio::test]
@@ -432,7 +569,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
.collect::<Result<_, _>>()?;
let task = create(&clients[0], &bucket, "cancelled");
wait_for_faults(&pauses).await?;
wait_for_faults(&pauses, &["WriteMetadata"]).await?;
task.abort();
assert!(task.await.expect_err("cancelled S3 future").is_cancelled());
drop(pauses);
@@ -541,7 +678,7 @@ async fn resume_at_rpc<T: Send + 'static>(
.collect::<Result<_, _>>()?;
eprintln!("MPU delay acceptance: methods={methods:?}, scope={scope}");
let task = tokio::spawn(request);
wait_for_faults(&pauses).await?;
wait_for_faults(&pauses, methods).await?;
assert!(
!task.is_finished(),
"the selected commit must wait while only one shard/voter is available"
@@ -618,6 +755,11 @@ async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestRe
let upload = client.create_multipart_upload().bucket(&bucket).key(&key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
let scope = upload_scope(upload_id)?;
let scope = if locks {
scope
} else {
upload_path_scope(&bucket, &key, upload_id)?
};
let request = client.abort_multipart_upload().bucket(&bucket).key(&key).upload_id(upload_id);
resume_at_rpc(&proxies, if locks { &["Lock", "LockBatch"] } else { &["Delete"] }, &scope, async move {
Ok(request.send().await?)
crates/e2e_test/src/rpc_fault_proxy.rs
+133 -6
View File
@@ -22,6 +22,8 @@ use hyper_util::{
rt::{TokioExecutor, TokioIo},
server::conn::auto,
};
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::DeleteRequest;
use std::{
io,
net::SocketAddr,
@@ -53,6 +55,7 @@ struct PauseState {
limit: usize,
matched: AtomicUsize,
blocked: AtomicUsize,
forwarded: AtomicUsize,
reached: Notify,
release: watch::Sender<Disposition>,
}
@@ -69,6 +72,11 @@ impl RpcPause {
self.state.blocked.load(Ordering::SeqCst)
}
/// Selected RPCs whose backend returned a response, excluding discarded packets.
pub(crate) fn forwarded(&self) -> usize {
self.state.forwarded.load(Ordering::SeqCst)
}
pub(crate) async fn wait_for_hits(&self, count: usize) -> io::Result<()> {
tokio::time::timeout(Duration::from_secs(20), async {
loop {
@@ -177,6 +185,7 @@ impl RpcFaultProxy {
limit,
matched: AtomicUsize::new(0),
blocked: AtomicUsize::new(0),
forwarded: AtomicUsize::new(0),
reached: Notify::new(),
release,
});
@@ -188,6 +197,30 @@ impl RpcFaultProxy {
}
}
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
if method != "Delete" {
return Ok(body.windows(scope.len()).any(|window| window == scope));
}
// Abort's recursive root delete must not be confused with transaction or
// part cleanup below that root. This is a unary, uncompressed gRPC frame.
if body.len() < 5 || body[0] != 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"scoped Delete needs an uncompressed unary frame",
));
}
let mut encoded_length = [0; 4];
encoded_length.copy_from_slice(&body[1..5]);
if u32::try_from(body.len() - 5).ok() != Some(u32::from_be_bytes(encoded_length)) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "scoped Delete frame length mismatch"));
}
let request = DeleteRequest::decode(&body[5..]).map_err(io::Error::other)?;
let options: serde_json::Value = serde_json::from_str(&request.options).map_err(io::Error::other)?;
Ok(request.volume == ".rustfs.sys/multipart"
&& request.path.trim_end_matches('/').as_bytes() == scope
&& options.get("recursive").and_then(serde_json::Value::as_bool) == Some(true))
}
async fn forward_request(
request: Request<Incoming>,
client: ProxyClient,
@@ -196,16 +229,17 @@ async fn forward_request(
) -> io::Result<hyper::Response<Incoming>> {
let (mut parts, body) = request.into_parts();
let mut body = body.map_err(io::Error::other).boxed();
let mut observed = None;
if let Some(pause) = pause {
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
if method.is_some_and(|name| pause.methods.contains(&name)) {
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
let matches = if let Some(scope) = pause.scope.as_deref() {
// Scope the control packet to this bucket or upload without
// decoding or changing bytes covered by the RPC signature.
// Inspect bounded routing data while retaining the exact
// original bytes covered by the RPC signature.
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let matches = bytes.windows(scope.len()).any(|window| window == scope);
let matches = scope_matches(method, &bytes, scope)?;
body = Full::new(bytes)
.map_err(|never| match never {})
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
@@ -230,6 +264,7 @@ async fn forward_request(
if disposition == Disposition::Discard {
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected RPC packet loss"));
}
observed = Some(pause);
}
}
}
@@ -247,10 +282,17 @@ async fn forward_request(
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
.build()
.map_err(io::Error::other)?;
client
let response = client
.request(Request::from_parts(parts, body))
.await
.map_err(io::Error::other)
.map_err(io::Error::other)?;
if let Some(pause) = observed
&& response.status().is_success()
&& response.headers().get("grpc-status").is_none_or(|status| status == "0")
{
pause.forwarded.fetch_add(1, Ordering::SeqCst);
}
Ok(response)
}
impl Drop for RpcFaultProxy {
@@ -404,6 +446,52 @@ mod tests {
.await
.expect("selective fault deadline");
}
#[tokio::test]
async fn released_observer_requires_a_selected_backend_response() {
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend listener");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy listener");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let observer = proxy
.pause(&["Lock"], usize::MAX, Some("recovery-bucket"))
.expect("RPC observer");
let unrelated = call_body(&client, proxy.local_addr(), "Lock", Bytes::from_static(b"other-bucket"))
.await
.expect("unrelated request");
unrelated.into_body().collect().await.expect("unrelated body");
assert_eq!(observer.forwarded(), 0, "unrelated traffic cannot prove recovery");
let task_client = client.clone();
let address = proxy.local_addr();
let request =
tokio::spawn(
async move { call_body(&task_client, address, "Lock", Bytes::from_static(b"recovery-bucket")).await },
);
observer.wait_for_hits(1).await.expect("selected request arrived");
assert_eq!(observer.forwarded(), 0, "an intercepted request has not reached its backend");
observer.release();
let body = request
.await
.expect("request task")
.expect("forwarded request")
.into_body()
.collect()
.await
.expect("forwarded body");
assert_eq!(body.trailers().expect("preserved trailers")["x-proof"], "unchanged-proof");
assert_eq!(body.to_bytes(), Bytes::from_static(b"recovery-bucket"));
assert_eq!(observer.forwarded(), 1);
let another = call_body(&client, address, "Lock", Bytes::from_static(b"recovery-bucket"))
.await
.expect("observer forwards without pausing");
another.into_body().collect().await.expect("second body");
assert_eq!(observer.forwarded(), 2);
})
.await
.expect("passive observer deadline");
}
#[tokio::test]
async fn scoped_fault_discards_expired_packets_without_replaying_them() {
tokio::time::timeout(Duration::from_secs(10), async {
@@ -442,4 +530,43 @@ mod tests {
.await
.expect("scoped discard deadline");
}
#[test]
fn delete_scope_matches_only_the_upload_root() {
let frame = |volume: &str, path: &str| {
let message = DeleteRequest {
volume: volume.to_owned(),
path: path.to_owned(),
options: r#"{"recursive":true}"#.to_owned(),
..Default::default()
};
let payload = message.encode_to_vec();
let length = u32::try_from(payload.len()).expect("small control packet");
let mut bytes = vec![0];
bytes.extend_from_slice(&length.to_be_bytes());
bytes.extend_from_slice(&payload);
bytes
};
let root = b"hash/upload";
assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload"), root).expect("root frame"));
assert!(scope_matches("Delete", &frame(".rustfs.sys/multipart", "hash/upload/"), root).expect("directory frame"));
for path in ["hash/upload/part.1", "hash/upload-other"] {
assert!(!scope_matches("Delete", &frame(".rustfs.sys/multipart", path), root).expect("other path"));
}
assert!(!scope_matches("Delete", &frame(".rustfs.sys/tmp", "hash/upload"), root).expect("other volume"));
let non_recursive = DeleteRequest {
volume: ".rustfs.sys/multipart".to_owned(),
path: "hash/upload".to_owned(),
options: r#"{"recursive":false}"#.to_owned(),
..Default::default()
}
.encode_to_vec();
let mut non_recursive_frame = vec![0];
non_recursive_frame.extend_from_slice(&u32::try_from(non_recursive.len()).expect("small frame").to_be_bytes());
non_recursive_frame.extend_from_slice(&non_recursive);
assert!(!scope_matches("Delete", &non_recursive_frame, root).expect("non-recursive frame"));
assert!(
scope_matches("Delete", &[0, 0, 0, 0, 1], root).is_err(),
"truncation must not become a match"
);
}
}
crates/ecstore/src/cache_value/metacache_set.rs
+36 -2
View File
@@ -40,7 +40,7 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
const EVENT_METACACHE_LISTING: &str = "metacache_listing";
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
pub type PartialFn =
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub(crate) type PartialResultFn = Box<
@@ -970,7 +970,7 @@ async fn list_path_raw_inner(
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(()),
_ = agreed_fn(current) => {}
result = agreed_fn(current) => result?
}
// warn!("list_path_raw: agreed_fn done");
}
@@ -1710,6 +1710,34 @@ mod tests {
.expect("aborting the listing should drop the blocked producer");
}
#[tokio::test]
async fn list_path_raw_propagates_agreed_callback_failure() {
let entry = MetaCacheEntry {
name: "bucket/object".to_owned(),
metadata: vec![1],
..Default::default()
};
let result = timeout(
Duration::from_secs(1),
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry])],
agreed: Some(Box::new(|_| Box::pin(async { Err(DiskError::ErasureReadQuorum) }))),
..Default::default()
},
),
)
.await
.expect("callback failure must terminate the walk");
assert_eq!(
result.expect_err("agreed resolution must not silently omit the key"),
DiskError::ErasureReadQuorum
);
}
#[tokio::test]
async fn list_path_raw_propagates_partial_callback_failure() {
let first = MetaCacheEntry {
@@ -1811,6 +1839,7 @@ mod tests {
Box::pin(async move {
callback_started.notify_one();
std::future::pending::<()>().await;
Ok(())
})
})),
..Default::default()
@@ -1962,6 +1991,7 @@ mod tests {
let seen = seen_clone.clone();
Box::pin(async move {
seen.lock().expect("seen mutex poisoned").push(entry.name);
Ok(())
})
})),
..Default::default()
@@ -2004,6 +2034,7 @@ mod tests {
let seen = seen_clone.clone();
Box::pin(async move {
seen.lock().expect("seen mutex poisoned").push(entry.name);
Ok(())
})
})),
..Default::default()
@@ -2076,6 +2107,7 @@ mod tests {
let seen = agreed_seen.clone();
Box::pin(async move {
seen.lock().expect("seen mutex poisoned").push(entry.name);
Ok(())
})
})),
partial: Some(Box::new(move |entries, _| {
@@ -2241,6 +2273,7 @@ mod tests {
let agreed_events = agreed_events.clone();
Box::pin(async move {
agreed_events.lock().expect("events lock").push((entry.name, 4usize, true));
Ok(())
})
})),
partial: Some(Box::new(move |entries, _| {
@@ -2397,6 +2430,7 @@ mod tests {
let name = entry.name;
Box::pin(async move {
agreed_obs.lock().expect("obs lock").push((name, vec![Some(desc); 4], true));
Ok(())
})
})),
partial: Some(Box::new(move |entries, _| {
crates/ecstore/src/core/pools.rs
+7 -1
View File
@@ -28644,7 +28644,13 @@ impl SetDisks {
min_disks: listing_quorum,
skip_walkdir_total_timeout: true,
walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
let future = cb1(entry);
Box::pin(async move {
future.await;
Ok(())
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
let resolver = resolver.clone();
let cb_func = cb_func.clone();
crates/ecstore/src/disk/local.rs
+141 -35
View File
@@ -13,6 +13,10 @@
// limitations under the License.
pub(in crate::disk) use self::commit::LocalRenamePreflightRejection;
#[cfg(test)]
#[path = "local/read_path_tests.rs"]
mod read_path_tests;
#[cfg(test)]
use self::commit::lock_rename_commit_directories;
@@ -115,6 +119,31 @@ use uuid::Uuid;
// Bound outstanding filesystem jobs and metadata buffers per disk request.
const PART_METADATA_READ_CONCURRENCY: usize = 8;
// Keep independent forward transactions bounded without reordering aliases,
// parent/child keys, rollback, or legacy operations.
const DELETE_VERSION_PREPARE_CONCURRENCY: usize = 8;
fn delete_version_batch_concurrency(versions: &[FileInfoVersions], opts: &DeleteOptions) -> usize {
if !cfg!(target_os = "linux") || !opts.recoverable_delete || opts.undo_write || opts.undo_delete || versions.len() < 2 {
return 1;
}
let mut names = HashSet::with_capacity(versions.len());
for version in versions {
let name = version.name.as_str();
if name.split('/').any(|segment| matches!(segment, "" | "." | "..")) || !names.insert(name) {
return 1;
}
}
for name in &names {
for (separator, _) in name.match_indices('/') {
if names.contains(&name[..separator]) {
return 1;
}
}
}
DELETE_VERSION_PREPARE_CONCURRENCY
}
/// The fresh health path uses a synchronous capacity syscall. Keep the permit
/// inside the blocking closure so cancelling its async caller cannot admit a
/// second probe while the first syscall is still running.
@@ -6775,8 +6804,19 @@ impl LocalDisk {
self.local_disk_bucket_and_object_path(self.io_root(), bucket, key)
}
fn io_get_checked_read_paths(&self, bucket: &str, key: &str) -> Result<(PathBuf, PathBuf)> {
let (bucket_path, path) = self.io_get_bucket_and_object_path(bucket, key)?;
#[cfg(test)]
read_path_tests::before_bucket_recheck();
// Reuse the buffer, not the validation result: retain the bucket
// recheck after the full object-path check and before queued IO.
self.check_valid_path(&bucket_path)?;
Ok((bucket_path, path))
}
/// Only for operations that open through PublicationRoot or a commit guard:
/// containment and symlink checks happen with the actual open, off the runtime.
/// Linux checks containment and symlinks at the actual open, off the runtime;
/// other platforms retain their preflight checks.
fn io_get_object_open_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
#[cfg(target_os = "linux")]
{
@@ -6789,7 +6829,8 @@ impl LocalDisk {
}
/// Only for operations that open through PublicationRoot or a commit guard:
/// containment and symlink checks happen with the actual open, off the runtime.
/// Linux checks containment and symlinks at the actual open, off the runtime;
/// other platforms retain their preflight checks.
fn io_get_bucket_and_object_open_path(&self, bucket: &str, key: &str) -> Result<(PathBuf, PathBuf)> {
#[cfg(target_os = "linux")]
{
@@ -6801,11 +6842,7 @@ impl LocalDisk {
Ok((bucket_path, path))
}
#[cfg(not(target_os = "linux"))]
{
let path = self.io_get_object_path(bucket, key)?;
let bucket_path = self.io_get_bucket_path(bucket)?;
Ok((bucket_path, path))
}
self.io_get_checked_read_paths(bucket, key)
}
fn io_get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
@@ -6879,10 +6916,6 @@ impl LocalDisk {
}
// Check if a path is valid
#[allow(
dead_code,
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
)]
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
#[cfg(target_os = "linux")]
{
@@ -7125,23 +7158,29 @@ impl LocalDisk {
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let mut errs = Vec::with_capacity(versions.len());
for _ in 0..versions.len() {
errs.push(None);
let concurrency = delete_version_batch_concurrency(&versions, &opts);
let mut errors = vec![None; versions.len()];
let mut pending = stream::iter(
versions
.into_iter()
.enumerate()
.map(|(index, version): (usize, FileInfoVersions)| {
let owner = namespace_owner.clone();
let options = &opts;
async move {
let error = self
.delete_versions_internal(volume, &version.name, &version.versions, options, owner)
.await
.err();
(index, error)
}
}),
)
.buffer_unordered(concurrency);
while let Some((index, error)) = pending.next().await {
errors[index] = error;
}
for (i, ver) in versions.iter().enumerate() {
if let Err(e) = self
.delete_versions_internal(volume, ver.name.as_str(), &ver.versions, &opts, namespace_owner.clone())
.await
{
errs[i] = Some(e);
} else {
errs[i] = None;
}
}
errs
errors
}
pub(crate) async fn delete_versions_guarded_with_namespace_owner(
@@ -10380,6 +10419,8 @@ fn local_disk_bucket_path(root: &Path, bucket: &str) -> Result<PathBuf> {
}
fn build_local_disk_bucket_path(root: &Path, bucket: &str) -> PathBuf {
#[cfg(test)]
read_path_tests::record_bucket_rebuild();
#[cfg(windows)]
let bucket_path = root.join(bucket.replace('/', "\\"));
#[cfg(not(windows))]
@@ -10389,8 +10430,8 @@ fn build_local_disk_bucket_path(root: &Path, bucket: &str) -> PathBuf {
}
fn check_local_disk_valid_object_path(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> {
let bucket_path = normalize_path_components(bucket_path);
let path = normalize_path_components(path);
let bucket_path = normalize_path_cow(bucket_path);
let path = normalize_path_cow(path);
if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) {
return Err(DiskError::InvalidPath);
}
@@ -10399,7 +10440,7 @@ fn check_local_disk_valid_object_path(root: &Path, bucket_path: &Path, path: &Pa
}
fn check_local_disk_valid_path(root: &Path, path: impl AsRef<Path>) -> Result<()> {
let path = normalize_path_components(path);
let path = normalize_path_cow(path.as_ref());
if !path.starts_with(root) {
return Err(DiskError::InvalidPath);
}
@@ -10430,7 +10471,7 @@ fn check_local_disk_object_path_components<'a>(
#[cfg(target_os = "linux")]
fn check_local_disk_valid_path_at(root: &Path, root_fd: &std::fs::File, path: impl AsRef<Path>) -> Result<()> {
let path = normalize_path_components(path);
let path = normalize_path_cow(path.as_ref());
if !path.starts_with(root) {
return Err(DiskError::InvalidPath);
}
@@ -10512,9 +10553,10 @@ fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()>
Ok(())
}
#[cfg(target_os = "linux")]
#[inline]
fn normalize_path_cow(path: &Path) -> std::borrow::Cow<'_, Path> {
// Windows prefixes retain the existing PathBuf reconstruction semantics.
#[cfg(not(windows))]
if !path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
@@ -10526,6 +10568,8 @@ fn normalize_path_cow(path: &Path) -> std::borrow::Cow<'_, Path> {
// Lightweight path normalization without filesystem calls
fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
#[cfg(test)]
read_path_tests::record_normalization();
let path = path.as_ref();
let mut result = PathBuf::new();
@@ -12382,7 +12426,11 @@ impl DiskAPI for LocalDisk {
.await?;
if let Some(flag) = opts.producer_limit_reached.as_ref() {
flag.store(limit_reached, std::sync::atomic::Ordering::Release);
// Every disk of every set in a page shares this flag, and the
// collector reads it as "did ANY bounded walk stop at its limit".
// A walk that reached EOF must not clear what another producer
// reported, whatever order they finish in.
flag.fetch_or(limit_reached, std::sync::atomic::Ordering::AcqRel);
}
out.close().await?;
Ok(())
@@ -13110,8 +13158,7 @@ impl LocalDisk {
Some(io_permit) => Some(io_permit),
None => acquire_context_disk_io_permit().await.map_err(DiskError::Io)?,
};
let file_path = self.io_get_object_path(volume, path)?;
let volume_dir = self.io_get_bucket_path(volume)?;
let (volume_dir, file_path) = self.io_get_checked_read_paths(volume, path)?;
let (data, _) = self
.read_all_data_with_dmtime(volume, volume_dir, file_path, io_permit)
.await?;
@@ -16187,6 +16234,65 @@ mod test {
);
}
// The collector decides "truncated or exhausted" from one flag shared by every
// producer of the page. It must stay set once any bounded walk stopped early,
// however the remaining producers finish.
#[tokio::test]
async fn walk_dir_keeps_shared_limit_flag_set_by_another_producer() {
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
let bucket = "test-bucket";
for idx in 0..3 {
let object_dir = dir.path().join(bucket).join(format!("prefix/object-{idx}"));
fs::create_dir_all(&object_dir).await.expect("object dir should be created");
fs::write(object_dir.join(STORAGE_FORMAT_FILE), b"meta")
.await
.expect("object metadata should be written");
}
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
disk.wait_for_startup_cleanup().await;
let walk = |limit: i32, flag: Arc<AtomicBool>| {
let disk = &disk;
async move {
let mut sink = Vec::new();
disk.walk_dir(
WalkDirOptions {
bucket: bucket.to_string(),
base_dir: "prefix/".to_string(),
recursive: true,
limit,
producer_limit_reached: Some(flag),
..Default::default()
},
&mut sink,
)
.await
.expect("walk should succeed");
}
};
let bounded = Arc::new(AtomicBool::new(false));
walk(1, bounded.clone()).await;
assert!(
bounded.load(Ordering::Acquire),
"a walk that stopped at its limit must report it through the shared flag"
);
// A sibling producer already stopped at its limit; this walk reaches EOF after it.
let shared = Arc::new(AtomicBool::new(true));
walk(0, shared.clone()).await;
assert!(
shared.load(Ordering::Acquire),
"a walk that reached EOF must not clear a limit another producer reported"
);
}
// #4644: the stall timeout is what catches a drive that stops answering.
//
// This exercises the bound directly rather than through `walk_dir`. A drive
crates/ecstore/src/disk/local/delete_recovery.rs
+181
View File
@@ -1262,6 +1262,17 @@ impl LocalDisk {
self.delete_recovery_barrier(volume, object, transaction, "PREACTIVATION")
.await?;
match fs::symlink_metadata(&destination).await {
Ok(metadata) if metadata.is_dir() => {
// Proof GC may persist before its final rmdir. Only an empty
// leaf can be retired; nested prefixes and new evidence fail
// closed under the same namespace ownership as publication.
match os::remove_dir_with_owner(&destination, owner.clone()).await {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) if is_dir_not_empty_error(&err) => return Err(DiskError::OutdatedXLMeta),
Err(err) => return Err(to_file_error(err).into()),
}
}
Ok(_) => return Err(DiskError::OutdatedXLMeta),
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(to_file_error(err).into()),
@@ -2278,6 +2289,131 @@ mod tests {
(temp, disk, info, bytes)
}
#[test]
fn delete_recovery_batch_parallelism_preserves_overlapping_key_order() {
let opts = DeleteOptions {
recoverable_delete: true,
..Default::default()
};
for names in [
vec!["a", "a"],
vec!["a", "a-0", "a/child"],
vec!["a//b", "a/b"],
vec!["a/./b", "a/b"],
vec!["a/../b", "b"],
] {
let groups: Vec<_> = names
.into_iter()
.map(|name| super::super::FileInfoVersions {
name: name.to_owned(),
..Default::default()
})
.collect();
assert_eq!(
super::super::delete_version_batch_concurrency(&groups, &opts),
1,
"overlapping or noncanonical keys retain serial execution"
);
}
let groups: Vec<_> = ["a/one", "a/two"]
.into_iter()
.map(|name| super::super::FileInfoVersions {
name: name.to_owned(),
..Default::default()
})
.collect();
let expected = if cfg!(target_os = "linux") {
super::super::DELETE_VERSION_PREPARE_CONCURRENCY
} else {
1
};
assert_eq!(super::super::delete_version_batch_concurrency(&groups, &opts), expected);
for options in [
DeleteOptions::default(),
DeleteOptions {
undo_write: true,
..opts.clone()
},
DeleteOptions {
undo_delete: true,
..opts
},
] {
assert_eq!(
super::super::delete_version_batch_concurrency(&groups, &options),
1,
"legacy and rollback ordering is unchanged"
);
}
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn delete_recovery_batch_prepares_independent_keys_while_one_metadata_lease_is_held() {
let (temp, disk, info, _) = fixture().await;
let disk = Arc::new(disk);
let mut healthy = info.clone();
healthy.name = "healthy".to_owned();
let healthy_root = temp.path().join("recovery/healthy");
fs::create_dir_all(&healthy_root).await.expect("healthy object directory");
fs::write(healthy_root.join(STORAGE_FORMAT_FILE), test_meta(healthy))
.await
.expect("healthy source");
let corrupt_root = temp.path().join("recovery/corrupt");
fs::create_dir_all(&corrupt_root).await.expect("corrupt object directory");
fs::write(corrupt_root.join(STORAGE_FORMAT_FILE), b"invalid xl.meta")
.await
.expect("corrupt source");
let held = os::acquire_metadata_mutation_lease(&temp.path().join("recovery/object"), None).await;
let groups = ["object", "corrupt", "healthy"]
.into_iter()
.map(|name| super::super::FileInfoVersions {
name: name.to_owned(),
versions: vec![FileInfo {
name: name.to_owned(),
..Default::default()
}],
..Default::default()
})
.collect();
let worker_disk = disk.clone();
let worker = tokio::spawn(async move {
worker_disk
.delete_versions_with_namespace_owner(
"recovery",
groups,
DeleteOptions {
recoverable_delete: true,
old_data_dir: Some(Uuid::new_v4()),
..Default::default()
},
None,
)
.await
});
let prepared = tokio::time::timeout(Duration::from_secs(5), async {
while fs::try_exists(healthy_root.join(STORAGE_FORMAT_FILE))
.await
.expect("inspect healthy metadata")
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(temp.path().join("recovery/object/xl.meta").exists(), "the locked source must stay intact");
drop(held);
let results = worker.await.expect("complete every disk result");
assert_eq!(
results,
vec![None, Some(DiskError::FileCorrupt), None],
"completion order cannot change per-key error positions"
);
assert!(
prepared.is_ok(),
"one key's metadata lease cannot serialize unrelated transaction preparation"
);
}
#[tokio::test]
async fn delete_recovery_absent_publish_refreshes_prepared_witness() {
let (_temp, disk, _info, _original) = fixture().await;
@@ -4062,6 +4198,51 @@ mod tests {
assert!(witness.original_digest.is_none() && witness.target_digest.is_none() && witness.data_dirs.is_empty());
}
#[tokio::test]
async fn delete_recovery_activation_preserves_occupied_transaction_directory() {
let (_source_temp, source, _, _) = fixture().await;
let transaction = Uuid::new_v4();
let applied = apply(&source, transaction).await;
let mut witness = source
.settle_delete_transaction(
"recovery",
"object",
transaction,
applied.fingerprint,
DeleteRecoveryAction::PublishCommit,
)
.await
.unwrap()
.journal
.unwrap();
witness.original_digest = None;
witness.target_digest = None;
witness.data_dirs.clear();
for nested in [false, true] {
let (_peer_temp, peer, _, _) = fixture().await;
let root = peer.io_get_object_path("recovery", "object").unwrap();
fs::remove_file(root.join(STORAGE_FORMAT_FILE)).await.unwrap();
let retained = root.join(transaction.to_string());
fs::create_dir(&retained).await.unwrap();
if nested {
fs::create_dir(retained.join("prefix-child")).await.unwrap();
} else {
fs::write(retained.join("unknown-file"), b"retained").await.unwrap();
}
assert!(matches!(
peer.activate_delete_preparation(&witness, None, None, None).await,
Err(DiskError::OutdatedXLMeta)
));
assert!(!retained.join(PREPARED).exists());
if nested {
assert!(retained.join("prefix-child").is_dir());
} else {
assert_eq!(fs::read(retained.join("unknown-file")).await.unwrap(), b"retained");
}
assert!(!root.join(STORAGE_FORMAT_FILE).exists());
}
}
#[tokio::test]
async fn delete_recovery_raw_discovery_keeps_malformed_records_and_later_keys() {
let (temp, disk, _, _) = fixture().await;
crates/ecstore/src/disk/local/read_path_tests.rs
+202
View File
@@ -0,0 +1,202 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use std::cell::RefCell;
struct Observation {
rebuilds: usize,
normalizations: usize,
before_recheck: Option<Box<dyn FnOnce()>>,
}
thread_local! {
static OBSERVATION: RefCell<Option<Observation>> = const { RefCell::new(None) };
}
struct Observe;
impl Observe {
fn new(before_recheck: Option<Box<dyn FnOnce()>>) -> Self {
OBSERVATION.with(|state| {
assert!(state.borrow().is_none(), "read-path observations must not nest");
*state.borrow_mut() = Some(Observation {
rebuilds: 0,
normalizations: 0,
before_recheck,
});
});
Self
}
fn normalizations(&self) -> usize {
OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").normalizations)
}
fn rebuilds(&self) -> usize {
OBSERVATION.with(|state| state.borrow().as_ref().expect("active observation").rebuilds)
}
}
impl Drop for Observe {
fn drop(&mut self) {
OBSERVATION.with(|state| *state.borrow_mut() = None);
}
}
pub(super) fn record_bucket_rebuild() {
OBSERVATION.with(|state| {
if let Some(observation) = state.borrow_mut().as_mut() {
observation.rebuilds += 1;
}
});
}
pub(super) fn record_normalization() {
OBSERVATION.with(|state| {
if let Some(observation) = state.borrow_mut().as_mut() {
observation.normalizations += 1;
}
});
}
pub(super) fn before_bucket_recheck() {
let hook = OBSERVATION.with(|state| state.borrow_mut().as_mut().and_then(|value| value.before_recheck.take()));
if let Some(hook) = hook {
hook();
}
}
async fn local_disk(root: &Path) -> LocalDisk {
let endpoint = Endpoint::try_from(root.to_string_lossy().as_ref()).expect("local endpoint");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
disk.make_volume("bucket").await.expect("bucket");
disk
}
#[tokio::test(flavor = "current_thread")]
async fn checked_read_paths_reuse_bucket_construction() {
let root = tempfile::tempdir().expect("disk root");
let disk = local_disk(root.path()).await;
std::fs::create_dir_all(root.path().join("bucket/prefix/object")).expect("object directories");
for key in ["prefix/object/xl.meta", "", "prefix/../object/xl.meta"] {
let expected = build_local_disk_object_path(disk.io_root(), "bucket", key);
let observation = Observe::new(None);
let actual = disk.io_get_checked_read_paths("bucket", key).expect("checked read paths");
assert_eq!(actual, expected, "returned paths must retain their existing spelling");
assert_eq!(observation.rebuilds(), 0, "the validated tuple already owns its bucket path");
// path_join_buf already resolves dot components in object keys.
let expected_normalizations = if cfg!(windows) { 3 } else { 0 };
assert_eq!(observation.normalizations(), expected_normalizations);
#[cfg(not(target_os = "linux"))]
{
assert_eq!(
disk.io_get_bucket_and_object_open_path("bucket", key)
.expect("read-version paths"),
expected
);
assert_eq!(observation.rebuilds(), 0, "read_version shares the same checked path core");
assert_eq!(observation.normalizations(), 2 * expected_normalizations);
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn checked_read_paths_preserve_containment_errors() {
let root = tempfile::tempdir().expect("disk root");
let disk = local_disk(root.path()).await;
for (bucket, key) in [("../outside", "object"), ("bucket", "../outside"), ("bucket", "/outside")] {
let old = disk
.io_get_object_path(bucket, key)
.and_then(|path| disk.io_get_bucket_path(bucket).map(|volume| (volume, path)));
let new = disk.io_get_checked_read_paths(bucket, key);
assert_eq!(new, old, "path and error behavior for {bucket}/{key}");
}
}
#[tokio::test(flavor = "current_thread")]
async fn checked_read_paths_metadata_bytes_and_absence_errors() {
let root = tempfile::tempdir().expect("disk root");
let disk = local_disk(root.path()).await;
let object = root.path().join("bucket/prefix/object");
std::fs::create_dir_all(&object).expect("object");
let metadata = b"opaque metadata bytes passed through the read API";
std::fs::write(object.join(STORAGE_FORMAT_FILE), metadata).expect("metadata");
{
let observation = Observe::new(None);
let actual = disk
.read_metadata("bucket", "prefix/object/xl.meta")
.await
.expect("read metadata");
assert_eq!(actual.as_ref(), metadata);
assert_eq!(observation.rebuilds(), 0, "scanner metadata reads use the shared path core");
assert_eq!(observation.normalizations(), if cfg!(windows) { 3 } else { 0 });
}
assert!(matches!(
disk.read_metadata("bucket", "missing/xl.meta").await,
Err(DiskError::FileNotFound)
));
assert!(matches!(
disk.read_metadata("missing", "object/xl.meta").await,
Err(DiskError::VolumeNotFound)
));
}
#[cfg(unix)]
#[tokio::test(flavor = "current_thread")]
async fn checked_read_paths_recheck_bucket_after_object_validation() {
let root = tempfile::tempdir().expect("disk root");
let outside = tempfile::tempdir().expect("outside root");
let disk = local_disk(root.path()).await;
std::fs::create_dir_all(root.path().join("bucket/object")).expect("object");
std::fs::write(root.path().join("bucket/object/xl.meta"), b"inside").expect("inside metadata");
let bucket = root.path().join("bucket");
let saved = root.path().join("saved");
let target = outside.path().to_path_buf();
let _observation = Observe::new(Some(Box::new(move || {
std::fs::rename(&bucket, saved).expect("replace bucket after full-path validation");
std::os::unix::fs::symlink(target, bucket).expect("bucket replacement symlink");
})));
assert!(
matches!(disk.io_get_checked_read_paths("bucket", "object/xl.meta"), Err(DiskError::InvalidPath)),
"the second bucket check must remain effective before queuing native IO"
);
}
#[test]
fn read_path_normalization_matches_component_normalizer() {
for value in [
"",
".",
"..",
"./a",
"a/./b",
"a//b/",
"a/../b",
"../../a",
"/",
"//a///b/",
"/a/../b",
"/../../a",
r"C:\a\b",
r"C:\a\..\b",
r"\\server\share\a",
r"\\?\C:\a\.\b",
] {
let path = Path::new(value);
let expected = normalize_path_components(path);
let observation = Observe::new(None);
let actual = normalize_path_cow(path);
assert_eq!(
actual.components().collect::<Vec<_>>(),
expected.components().collect::<Vec<_>>(),
"{value}"
);
let needs_buffer = cfg!(windows)
|| path
.components()
.any(|component| matches!(component, std::path::Component::ParentDir | std::path::Component::CurDir));
assert_eq!(matches!(actual, std::borrow::Cow::Owned(_)), needs_buffer, "{value}");
assert_eq!(observation.normalizations(), usize::from(needs_buffer), "{value}");
}
}
crates/ecstore/src/services/rebalance/worker.rs
+7 -1
View File
@@ -641,7 +641,13 @@ impl SetDisks {
state = "agreed_entry",
"Rebalance listing agreed entry"
);
Box::pin(cb1(entry))
{
let future = cb1(entry);
Box::pin(async move {
future.await;
Ok(())
})
}
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
let resolver = resolver.clone();
crates/ecstore/src/set_disk/core/metadata_read.rs
+2
View File
@@ -1082,6 +1082,8 @@ impl SetDisks {
let response_start = observe.then(Instant::now);
let result = if let Some(disk) = disk {
Self::record_read_version_call(&object, disk_index);
#[cfg(test)]
Self::read_version_fanout_barrier(&object, disk_index).await;
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
Self::record_metadata_slowtail_fault(&object, disk_index);
tokio::time::sleep(delay).await;
crates/ecstore/src/set_disk/ops/delete_recovery.rs
+50
View File
@@ -1783,6 +1783,56 @@ mod tests {
.expect("returning old replica must not leave bucket residue");
}
#[tokio::test]
async fn delete_recovery_set_replays_partial_gc_with_empty_transaction_directory() {
let bucket = "recover-empty-gc";
let (dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &[24; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
for (index, disk) in disks.iter().enumerate() {
publish(disk, bucket, transaction).await;
let state = disk.inspect_delete_transaction(bucket, "object", transaction).await.unwrap();
let finished = disk
.settle_delete_transaction(bucket, "object", transaction, state.fingerprint, DeleteRecoveryAction::FinishCommit)
.await
.unwrap();
if index > 0 {
disk.settle_delete_transaction(
bucket,
"object",
transaction,
finished.fingerprint,
DeleteRecoveryAction::FinalizeCommit,
)
.await
.unwrap();
}
}
// A crash can persist proof removal before the final empty rmdir.
let empty = dirs[3].path().join(bucket).join("object").join(transaction.to_string());
tokio::fs::create_dir_all(&empty).await.unwrap();
for _ in 0..2 {
store
.recover_delete_transactions(bucket, &HealOpts::default())
.await
.expect("the remaining receipt must settle every participant without a fresh delete");
for disk in &disks {
assert!(discover(disk, bucket, Some("object")).await.unwrap().is_empty());
assert!(
disk.inspect_delete_transaction(bucket, "object", transaction)
.await
.unwrap()
.current_digest
.is_none()
);
}
}
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("ordinary bucket removal must succeed after replay");
}
#[tokio::test]
async fn delete_recovery_tracks_only_observed_removed_version() {
let bucket = "recover-actual-removal";
crates/ecstore/src/set_disk/ops/heal_walk.rs
+1
View File
@@ -358,6 +358,7 @@ impl SetDisks {
let collector = agreed_collector.clone();
Box::pin(async move {
collector.ingest(entry);
Ok(())
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _errs: &[Option<DiskError>]| {
crates/ecstore/src/set_disk/ops/multipart.rs
+101 -4
View File
@@ -1264,9 +1264,8 @@ impl SetDisks {
upload_id_path: &str,
write: bool,
) -> Result<(FileInfo, Vec<FileInfo>)> {
let disks = self.disks.read().await;
let disks = disks.clone();
// Slow metadata I/O must not hold the topology registry's read lock.
let disks = self.get_disks_internal().await;
let (parts_metadata, errs) =
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, upload_id_path, "", false, false, false)
@@ -2656,8 +2655,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let _upload_guard = self
.acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts)
.await?;
// A failed completion can consume staging metadata on some disks.
// Identify the upload with read quorum; deletion below still requires
// its full write quorum and the upload's incarnation/lifecycle fences.
let (fi, _) = self
.check_upload_id_exists_with_opts(bucket, object, upload_id, true, opts)
.check_upload_id_exists_with_opts(bucket, object, upload_id, false, opts)
.await?;
ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
@@ -7599,6 +7601,101 @@ mod tests {
.await;
}
#[tokio::test]
async fn multipart_metadata_io_does_not_block_disk_registry_updates() {
use crate::set_disk::core::io_primitives::rename_fanout_barrier::{self, PHASE_READ_VERSION};
let (_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-metadata-registry";
let object = "object";
make_bucket_on_all(&disks, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("create upload for metadata lookup");
let path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
for write in [false, true] {
let barrier = rename_fanout_barrier::arm(&path, 0, PHASE_READ_VERSION);
let lookup_set = set_disks.clone();
let upload_id = upload.upload_id.clone();
let lookup = tokio::spawn(async move { lookup_set.check_upload_id_exists(bucket, object, &upload_id, write).await });
tokio::time::timeout(Duration::from_secs(5), barrier.wait_until_paused())
.await
.expect("metadata I/O reaches its deterministic barrier");
assert!(!lookup.is_finished(), "metadata lookup must still be waiting on disk I/O");
let registry_writable = tokio::time::timeout(Duration::from_secs(2), set_disks.disks.write())
.await
.is_ok();
barrier.release();
lookup
.await
.expect("metadata lookup task")
.expect("metadata quorum remains valid");
assert!(
registry_writable,
"disk registry writes must proceed while metadata I/O is paused, write={write}"
);
}
}
#[tokio::test]
async fn abort_recovers_read_quorum_staging_metadata() {
for remaining in [2, 1] {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-abort-partial-commit";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("create upload before partial commit");
put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x42; 4096], 4096).await;
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
// A failed completion can consume staging metadata on the disks
// whose rename succeeded while the remaining disks retain it.
for directory in &temp_dirs[remaining..] {
tokio::fs::remove_file(
directory
.path()
.join(RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_path)
.join("xl.meta"),
)
.await
.expect("consume staging metadata without deleting residual parts");
}
let result = set_disks
.abort_multipart_upload(bucket, object, &upload.upload_id, &ObjectOptions::default())
.await;
if remaining == 2 {
result.expect("read quorum identifies the upload; delete still requires write quorum");
for directory in &temp_dirs {
assert!(
!directory
.path()
.join(RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_path)
.exists(),
"abort must reclaim staging on every available disk"
);
}
} else {
assert!(matches!(result, Err(StorageError::InvalidUploadID(..))), "{result:?}");
for directory in &temp_dirs {
assert!(
directory
.path()
.join(RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_path)
.exists(),
"metadata below read quorum must not authorize deletion"
);
}
}
}
}
#[tokio::test]
async fn abort_enforces_delete_write_quorum_boundary() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
crates/ecstore/src/store/delete_recovery_runtime.rs
+26
View File
@@ -525,6 +525,12 @@ impl RecoveryPass {
}
async fn discover(&mut self, store: &Arc<ECStore>, cancel: &CancellationToken) {
// ECStore starts before the bucket metadata system is published. Do
// not turn that bootstrap state into a deferred disk/bucket failure.
if store.ctx.bucket_metadata_sys().is_none() {
return;
}
let hints = store.ctx.delete_recovery_hints();
// Renew only this worker's bounded local cursors before any await.
// Backpressure and a slow canonical rollback must not erase progress.
@@ -1032,6 +1038,26 @@ mod tests {
});
}
#[tokio::test]
async fn delete_recovery_waits_for_metadata_before_consuming_startup_discovery() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
store.background_cancel_token().expect("background token").cancel();
tokio::task::yield_now().await;
assert!(store.ctx.bucket_metadata_sys().is_none(), "metadata bootstrap has not completed");
let hints = store.ctx.delete_recovery_hints();
hints.request_rescan();
let generation = hints.rescan().expect("startup discovery obligation");
let mut pass = RecoveryPass::default();
pass.discover(&store, &CancellationToken::new()).await;
assert_eq!(hints.rescan(), Some(generation), "bootstrap cannot consume the discovery obligation");
assert!(pass.deferred.is_empty(), "bootstrap is not a failed disk scope");
assert!(pass.next_inventory.is_none(), "bootstrap must not delay the initial inventory");
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
pass.discover(&store, &CancellationToken::new()).await;
assert!(hints.rescan().is_none(), "the first ready pass fulfills the empty-store scan immediately");
assert!(pass.deferred.is_empty());
}
#[tokio::test]
async fn delete_recovery_clean_startup_discovers_hidden_preparation() {
let bucket = "runtime-clean-start";
crates/ecstore/src/store/init.rs
+90 -6
View File
@@ -11861,6 +11861,90 @@ mod tests {
.len()
}
// Recovery removes metadata replicas disk by disk. An unreadable prefix
// is incomplete evidence and must reset, never advance, the empty streak.
#[cfg(feature = "test-util")]
fn tier_delete_recovery_listing_is_empty(prefix: &str, count: crate::error::Result<usize>) -> bool {
match count {
Ok(count) => count == 0,
Err(StorageError::InsufficientReadQuorum(bucket, object)) if bucket == RUSTFS_META_BUCKET && object == prefix => {
false
}
Err(error) => panic!("{prefix} recovery inventory should be listable: {error:?}"),
}
}
#[cfg(feature = "test-util")]
async fn tier_delete_recovery_prefix_is_empty(store: Arc<crate::store::ECStore>, prefix: &str) -> bool {
let store_id = store.id;
let started = std::time::Instant::now();
let count = store
.list_objects_v2(RUSTFS_META_BUCKET, prefix, None, None, 100, false, None, false)
.await
.map(|listing| listing.objects.len());
if let Err(error) = &count {
eprintln!(
"tier-delete recovery inventory unavailable: store={store_id} prefix={prefix} elapsed={:?} error={error:?}",
started.elapsed()
);
}
tier_delete_recovery_listing_is_empty(prefix, count)
}
#[cfg(feature = "test-util")]
#[test]
fn tier_delete_recovery_observations_require_readable_empty_prefixes() {
for prefix in [TIER_DELETE_JOURNAL_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_PREFIX] {
let observations = [
Ok(0),
Err(StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), prefix.to_string())),
Ok(0),
Ok(1),
Ok(0),
];
let mut stable_empty = 0;
let streaks = observations
.into_iter()
.map(|count| {
if tier_delete_recovery_listing_is_empty(prefix, count) {
stable_empty += 1;
} else {
stable_empty = 0;
}
stable_empty
})
.collect::<Vec<_>>();
assert_eq!(streaks, [1, 0, 1, 0, 1], "unreadable or nonempty {prefix} cannot certify convergence");
for _ in 0..10 {
assert!(!tier_delete_recovery_listing_is_empty(
prefix,
Err(StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), prefix.to_string())),
));
}
}
}
#[cfg(feature = "test-util")]
#[test]
fn tier_delete_recovery_observations_reject_unrelated_errors() {
for prefix in [TIER_DELETE_JOURNAL_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_PREFIX] {
for error in [
StorageError::InsufficientReadQuorum("other-bucket".to_string(), prefix.to_string()),
StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "other-prefix".to_string()),
StorageError::FileNotFound,
StorageError::ErasureReadQuorum,
] {
assert!(
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
tier_delete_recovery_listing_is_empty(prefix, Err(error))
}))
.is_err(),
"only the exact recovery prefix's InsufficientReadQuorum may defer an observation"
);
}
}
}
#[cfg(feature = "test-util")]
async fn install_committed_tier_delete_journals(
store: Arc<crate::store::ECStore>,
@@ -11968,8 +12052,8 @@ mod tests {
recover_tier_delete_journal_entries(store.clone(), 100, None)
.await
.expect("restarted journal recovery pass should finish");
if tier_delete_journal_count(store.clone()).await == 0
&& tier_delete_dispatch_manifest_count(store.clone()).await == 0
if tier_delete_recovery_prefix_is_empty(store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
&& tier_delete_recovery_prefix_is_empty(store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
&& tier_delete_dispatch_manifest_recovery_count_for_test(&store) == 0
{
stable_empty_observations += 1;
@@ -13397,8 +13481,8 @@ mod tests {
tokio::time::timeout(Duration::from_secs(30), async {
let mut stable_empty_observations = 0;
loop {
if tier_delete_journal_count(restarted_store.clone()).await == 0
&& tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
if tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
&& tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
&& tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
{
stable_empty_observations += 1;
@@ -13522,8 +13606,8 @@ mod tests {
tokio::time::timeout(Duration::from_secs(30), async {
let mut stable_empty_observations = 0;
loop {
if tier_delete_journal_count(restarted_store.clone()).await == 0
&& tier_delete_dispatch_manifest_count(restarted_store.clone()).await == 0
if tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_JOURNAL_PREFIX).await
&& tier_delete_recovery_prefix_is_empty(restarted_store.clone(), TIER_DELETE_DISPATCH_MANIFEST_PREFIX).await
&& tier_delete_dispatch_manifest_recovery_count_for_test(&restarted_store) == 0
{
stable_empty_observations += 1;
crates/ecstore/src/store/list_objects.rs
+457 -348
View File
File diff suppressed because it is too large Load Diff
crates/filemeta/benches/xl_meta_bench.rs
+187 -1
View File
@@ -596,6 +596,191 @@ fn bench_metacache_resolve(c: &mut Criterion) {
group.finish();
}
fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry {
let mut metadata = FileMeta::new();
for index in 0..versions {
let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) };
let mut info = FileInfo::new("object", data, parity);
info.volume = "bucket".to_owned();
info.erasure.index = 1;
info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index")));
info.versioned = true;
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index")));
info.mod_time = Some(
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"),
);
info.size = 1;
info.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
info.metadata.insert("etag".to_owned(), format!("etag-{index}"));
info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024));
metadata.add_version(info).expect("valid version history");
}
MetaCacheEntry {
name: "object".to_owned(),
metadata: metadata.marshal_msg().expect("serialize history"),
cached: cached.then_some(metadata),
reusable: false,
}
}
#[inline]
fn owned_agreed_latest_reference(
mut entry: MetaCacheEntry,
replicas: usize,
params: &rustfs_filemeta::MetadataResolutionParams,
enforce_write_quorum: bool,
) -> rustfs_filemeta::MetaCacheEntryResolution {
use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
if !enforce_write_quorum {
return Resolved(entry);
}
if entry.is_dir() {
return if replicas >= params.dir_quorum {
Resolved(entry)
} else {
Rejected
};
}
if replicas < params.obj_quorum {
return Rejected;
}
let metadata = match entry.xl_meta_ref() {
Ok(metadata) => metadata,
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
Err(_) => return Rejected,
};
if params.requested_versions == 1
&& let Some(first) = metadata.versions.first()
&& !first.header.free_version()
&& replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack)
{
return Resolved(entry);
}
// These fixtures cover the fast prefix, never older-version reconciliation.
NeedsSupplement(entry)
}
fn bench_listing_borrowed_metadata(c: &mut Criterion) {
use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams};
let mut group = c.benchmark_group("listing_borrowed_metadata");
group
.sample_size(20)
.warm_up_time(Duration::from_millis(150))
.measurement_time(Duration::from_millis(700));
for versions in [2, 16, 256] {
for cached in [false, true] {
let entry = listing_history(versions, cached);
let params = MetadataResolutionParams {
dir_quorum: 8,
obj_quorum: 8,
requested_versions: 1,
write_quorum_slack: 8,
bucket: "bucket".to_owned(),
..Default::default()
};
let slots = vec![Some(&entry); 8];
let owned = MetaCacheEntries(vec![Some(entry.clone()); 8])
.resolve_with_write_quorum(params.clone())
.expect("previous committed version");
let borrowed =
MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version");
assert_eq!(owned.metadata, borrowed.metadata);
let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" });
group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size")));
// ABBA order permits an explicit baseline-drift check for each fixture.
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| {
b.iter(|| {
if phase.starts_with("owned") {
black_box(
MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8])
.resolve_with_write_quorum(params.clone()),
)
} else {
let slots = vec![Some(black_box(&entry)); 8];
black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()))
}
});
});
}
let mut agreed_metadata = entry
.cached
.clone()
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture"));
let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body");
let object = latest.object.as_mut().expect("latest object");
object.erasure_m = 8;
object.erasure_n = 8;
agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header");
let agreed = MetaCacheEntry {
name: entry.name.clone(),
metadata: agreed_metadata.marshal_msg().expect("agreed history"),
cached: cached.then_some(agreed_metadata),
reusable: false,
};
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| {
b.iter_batched(
|| agreed.clone(),
|candidate| {
let resolved = if phase.starts_with("owned") {
owned_agreed_latest_reference(candidate, 8, &params, true)
} else {
rustfs_filemeta::resolve_agreed_entry(candidate, 8, &params, true)
};
let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else {
panic!("balanced latest must use agreed fast prefix");
};
black_box(candidate)
},
BatchSize::SmallInput,
);
});
}
let replicas: Vec<_> = (1..=8)
.map(|disk| {
let mut metadata = entry
.cached
.clone()
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture"));
for version in &mut metadata.versions {
let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body");
payload.object.as_mut().expect("object body").erasure_index = disk;
version.meta = payload.marshal_msg().expect("per-disk body");
}
MetaCacheEntry {
name: entry.name.clone(),
metadata: metadata.marshal_msg().expect("per-disk metadata"),
cached: cached.then_some(metadata),
reusable: false,
}
})
.collect();
assert!(replicas[0].headers_agree(&replicas, false));
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| {
b.iter(|| {
let agreed = if phase.starts_with("owned") {
replicas
.iter()
.all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1)
} else {
replicas[0].headers_agree(black_box(&replicas), false)
};
black_box(agreed)
});
});
}
}
}
group.finish();
}
criterion_group!(
benches,
bench_create_real_xlmeta,
@@ -612,7 +797,8 @@ criterion_group!(
bench_list_page_to_fileinfo,
bench_delete_inline_versions,
bench_metacache_stream_page,
bench_metacache_resolve
bench_metacache_resolve,
bench_listing_borrowed_metadata
);
criterion_main!(benches);
crates/filemeta/src/filemeta.rs
+1
View File
@@ -155,6 +155,7 @@ pub fn is_skip_meta_key(key: &str) -> bool {
}
mod codec;
pub(crate) use codec::{BorrowedFileMeta, reconcile_headers};
mod inline_data;
pub(crate) mod msgp_decode;
mod validation;
crates/filemeta/src/filemeta/codec.rs
+4 -1
View File
@@ -597,7 +597,7 @@ impl FileMeta {
return false;
}
let mut is_delete_marker = false;
let mut is_delete_marker = true;
let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| {
let mut header = FileMetaVersionHeader::default();
@@ -605,6 +605,9 @@ impl FileMeta {
return Err(Error::DoneForNow);
}
if header.free_version() {
return Ok(());
}
is_delete_marker = header.version_type == VersionType::Delete;
Err(Error::DoneForNow)
crates/filemeta/src/filemeta/version.rs
+82 -11
View File
@@ -646,6 +646,11 @@ impl FileMetaShallowVersion {
write_quorum_from_header_and_meta(&self.header, self.meta.as_slice(), fallback_quorum)
}
/// Quorum for listing this version without accepting an unreadable payload.
pub fn listing_quorum(&self, fallback_quorum: usize, unreachable_disks: usize) -> usize {
listing_quorum_from_header_and_meta(&self.header, &self.meta, fallback_quorum, unreachable_disks)
}
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
FileMetaVersion::decode_fileinfo_from_slice(self.meta.as_slice(), volume, path, all_parts, true)
}
@@ -664,6 +669,33 @@ pub(crate) fn write_quorum_from_header_and_meta(header: &FileMetaVersionHeader,
.unwrap_or(header_quorum)
}
pub(crate) fn listing_quorum_from_header_and_meta(
header: &FileMetaVersionHeader,
meta: &[u8],
fallback_quorum: usize,
unreachable_disks: usize,
) -> usize {
let geometry = if header.version_type == VersionType::Object && header.has_ec() {
Some((usize::from(header.ec_m), usize::from(header.ec_n)))
} else if matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
FileMetaVersion::try_from(meta)
.ok()
.and_then(|version| version.erasure_geometry())
} else {
None
};
let Some((data_blocks, parity_blocks)) = geometry else {
return fallback_quorum;
};
let Some(write_quorum) = write_quorum_from_erasure(data_blocks, parity_blocks) else {
return fallback_quorum;
};
write_quorum
.saturating_sub(unreachable_disks)
.max(data_blocks)
.max(fallback_quorum)
}
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
type Error = Error;
@@ -706,8 +738,8 @@ pub(super) struct V2ObjectSliceSummary {
}
impl FileMetaVersion {
fn erasure_write_quorum(&self) -> Option<usize> {
let (data_blocks, parity_blocks) = match self.version_type {
fn erasure_geometry(&self) -> Option<(usize, usize)> {
Some(match self.version_type {
VersionType::Object | VersionType::Legacy => {
if let Some(object) = &self.object {
(object.erasure_m, object.erasure_n)
@@ -718,8 +750,11 @@ impl FileMetaVersion {
}
}
_ => return None,
};
})
}
fn erasure_write_quorum(&self) -> Option<usize> {
let (data_blocks, parity_blocks) = self.erasure_geometry()?;
write_quorum_from_erasure(data_blocks, parity_blocks)
}
@@ -4183,11 +4218,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
let required_quorum = streams[0]
.first()
.map(|&(hdr, meta)| {
write_quorum_from_header_and_meta(hdr, meta, quorum)
.saturating_sub(write_quorum_slack)
.max(quorum)
})
.map(|&(hdr, meta)| listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack))
.unwrap_or(quorum);
if streams.len() >= required_quorum {
return streams[0].iter().copied().map(to_shallow).collect();
@@ -4201,9 +4232,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
let required_quorum = |(hdr, meta): (&FileMetaVersionHeader, &[u8])| {
if enforce_write_quorum {
write_quorum_from_header_and_meta(hdr, meta, quorum)
.saturating_sub(write_quorum_slack)
.max(quorum)
listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack)
} else {
quorum
}
@@ -5176,6 +5205,48 @@ mod tests {
assert_eq!(version.write_quorum(5), 7);
}
#[test]
fn listing_quorum_uses_each_versions_data_floor() {
for (data, parity, offline, fallback, expected) in
[(8, 8, 0, 9, 9), (8, 8, 8, 8, 8), (12, 4, 8, 8, 12), (12, 4, 0, 9, 12)]
{
let version = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Object,
ec_m: data,
ec_n: parity,
..Default::default()
},
// Modern headers must not decode the payload to decide quorum.
meta: vec![0xff],
};
assert_eq!(version.listing_quorum(fallback, offline), expected);
}
let marker = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Delete,
..Default::default()
},
meta: Vec::new(),
};
assert_eq!(marker.listing_quorum(9, 0), 9, "healthy marker needs majority metadata");
assert_eq!(marker.listing_quorum(8, 8), 8, "degraded marker retains metadata floor");
}
#[test]
fn listing_quorum_preserves_legacy_and_zero_parity_data_floor() {
for (data, parity, fallback, expected) in [(12, 4, 8, 12), (8, 8, 8, 8), (4, 0, 2, 4)] {
let version = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Object,
..Default::default()
},
meta: encode_legacy_v2_object_body(data, parity),
};
assert_eq!(version.listing_quorum(fallback, 8), expected);
}
}
#[test]
fn shallow_version_write_quorum_uses_zero_parity_object_payload() {
let expected = sample_header();
crates/filemeta/src/metacache.rs
+497 -36
View File
@@ -53,8 +53,8 @@ pub struct MetadataResolutionParams {
pub strict: bool,
/// Number of set drives that were unreachable when the listing snapshot was
/// taken. Write-quorum enforcement relaxes each version's required quorum by
/// this amount (never below `obj_quorum`): a version legally committed at
/// write quorum can have that many of its metadata copies on drives no
/// this amount, never below the version's data quorum or `obj_quorum`. A
/// version legally committed at write quorum can have metadata on drives no
/// reader could consult, and must not be dropped for it.
pub write_quorum_slack: usize,
pub candidates: Vec<Vec<FileMetaShallowVersion>>,
@@ -139,14 +139,13 @@ impl MetaCacheEntry {
return idx.is_none() || idx.unwrap() == self.name.len() - separator.len();
}
let ext = self.name.trim_start_matches(dir);
// Strip `dir` exactly once: a repeated strip would treat `a/a/b` as a
// direct child of `a/`.
let Some(ext) = self.name.strip_prefix(dir) else {
return false;
};
if ext.len() != self.name.len() {
let idx = ext.find(separator);
return idx.is_none() || idx.unwrap() == ext.len() - separator.len();
}
false
ext.find(separator).is_none_or(|idx| idx == ext.len() - separator.len())
}
pub fn is_object(&self) -> bool {
@@ -159,10 +158,11 @@ impl MetaCacheEntry {
pub fn is_latest_delete_marker(&mut self) -> bool {
if let Some(cached) = &self.cached {
if cached.versions.is_empty() {
return true;
}
return cached.versions[0].header.version_type == VersionType::Delete;
return cached
.versions
.iter()
.find(|version| !version.header.free_version())
.is_none_or(|version| version.header.version_type == VersionType::Delete);
}
if !FileMeta::is_xl2_v1_format(&self.metadata) {
@@ -179,12 +179,11 @@ impl MetaCacheEntry {
}
match self.xl_meta() {
Ok(res) => {
if res.versions.is_empty() {
return true;
}
res.versions[0].header.version_type == VersionType::Delete
}
Ok(res) => res
.versions
.iter()
.find(|version| !version.header.free_version())
.is_none_or(|version| version.header.version_type == VersionType::Delete),
Err(_) => true,
}
}
@@ -277,6 +276,88 @@ impl MetaCacheEntry {
FileMeta::decode_file_info_versions_from_buf(&self.metadata, bucket, self.name.as_str(), false)
}
fn borrow_metadata(&self) -> Result<crate::filemeta::BorrowedFileMeta<'_>> {
if self.is_dir() {
return Err(Error::FileNotFound);
}
match &self.cached {
Some(metadata) => Ok(metadata.borrow_cached()),
None => FileMeta::decode_borrowed(&self.metadata),
}
}
/// Compare a replica group after validating this history once.
pub fn headers_agree<'a>(&'a self, replicas: impl IntoIterator<Item = &'a Self>, strict: bool) -> bool {
if self.is_dir() {
return replicas.into_iter().all(|other| self.name == other.name && other.is_dir());
}
let Ok(metadata) = self.borrow_metadata() else {
return false;
};
replicas.into_iter().all(|other| {
if self.name != other.name || other.is_dir() {
return false;
}
if std::ptr::eq(self, other) {
return true;
}
if let Some(other) = &other.cached {
crate::filemeta::reconcile_headers(
other.versions.iter().map(|version| &version.header),
metadata.versions.as_slice().iter().map(|(header, _, _)| header),
true,
strict,
)
.is_some()
} else {
FileMeta::decode_borrowed_with_hint(&other.metadata, Some(&metadata)).is_ok_and(|candidate| {
crate::filemeta::reconcile_headers(
candidate.versions.as_slice().iter().map(|(header, _, _)| header),
metadata.versions.as_slice().iter().map(|(header, _, _)| header),
true,
strict,
)
.is_some()
})
}
})
}
fn cached_latest_listing_quorum(&self, params: &MetadataResolutionParams) -> Option<usize> {
if params.requested_versions != 1 {
return None;
}
self.cached
.as_ref()?
.versions
.first()
.filter(|version| !version.header.free_version())
.map(|version| version.listing_quorum(params.obj_quorum, params.write_quorum_slack))
}
/// Whether the inspected versions need more independent replica votes.
pub fn needs_quorum_supplement(&self, replicas: usize, params: &MetadataResolutionParams) -> Result<bool> {
if let Some(required) = self.cached_latest_listing_quorum(params) {
return Ok(replicas < required);
}
let metadata = self.borrow_metadata()?;
let mut selected = 0;
for (header, _, meta) in metadata.versions.as_slice() {
let required =
crate::filemeta::listing_quorum_from_header_and_meta(header, meta, params.obj_quorum, params.write_quorum_slack);
if replicas < required && (params.requested_versions == 0 || selected < params.requested_versions) {
return Ok(true);
}
if !header.free_version() {
selected += 1;
}
if params.requested_versions > 0 && selected == params.requested_versions {
break;
}
}
Ok(false)
}
pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
if other.is_none() {
return (None, false);
@@ -408,6 +489,125 @@ impl MetaCacheEntry {
}
}
/// Outcome of resolving one history observed identically by independent readers.
pub enum MetaCacheEntryResolution {
Resolved(MetaCacheEntry),
NeedsSupplement(MetaCacheEntry),
Rejected,
}
/// Apply per-version listing quorum without materializing unchanged histories.
#[inline]
pub fn resolve_agreed_entry(
mut entry: MetaCacheEntry,
replicas: usize,
params: &MetadataResolutionParams,
enforce_write_quorum: bool,
) -> MetaCacheEntryResolution {
use MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
if !enforce_write_quorum {
return Resolved(entry);
}
if entry.is_dir() {
return if replicas >= params.dir_quorum {
Resolved(entry)
} else {
Rejected
};
}
if replicas < params.obj_quorum {
return Rejected;
}
if let Some(required) = entry.cached_latest_listing_quorum(params) {
return if replicas >= required {
Resolved(entry)
} else {
NeedsSupplement(entry)
};
}
let metadata = match entry.borrow_metadata() {
Ok(metadata) => metadata,
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
Err(_) => return Rejected,
};
let versions = metadata.versions.as_slice();
let mut selected = Vec::new();
let mut selected_objects = 0;
let mut needs_supplement = false;
for (index, (header, _, meta)) in versions.iter().enumerate() {
let required =
crate::filemeta::listing_quorum_from_header_and_meta(header, meta, params.obj_quorum, params.write_quorum_slack);
if replicas < required {
needs_supplement |= params.requested_versions == 0 || selected_objects < params.requested_versions;
continue;
}
// The common latest-user-version path keeps the original buffer and cache.
if index == 0 && params.requested_versions == 1 && !header.free_version() {
return Resolved(entry);
}
selected.push(index);
if !header.free_version() {
selected_objects += 1;
}
if params.requested_versions > 0 && selected_objects == params.requested_versions {
break;
}
}
if needs_supplement {
return NeedsSupplement(entry);
}
if selected.is_empty() {
return Rejected;
}
if selected.len() == versions.len() {
return Resolved(entry);
}
if entry.cached.is_some() {
drop(metadata);
let Some(cached) = entry.cached.as_mut() else {
return Rejected;
};
let mut selected = selected.into_iter().peekable();
let mut index = 0;
cached.versions.retain(|_| {
let keep = selected.peek() == Some(&index);
if keep {
selected.next();
}
index += 1;
keep
});
let Ok(metadata) = cached.marshal_msg() else {
return Rejected;
};
entry.metadata = metadata;
} else {
let mut filtered = FileMeta {
meta_ver: metadata.meta_ver,
..Default::default()
};
if !metadata.inline_buf.is_empty() {
filtered.data.update(metadata.inline_buf);
}
filtered.versions = selected
.into_iter()
.map(|index| {
let (header, _, meta) = &versions[index];
FileMetaShallowVersion {
header: header.clone(),
meta: meta.to_vec(),
}
})
.collect();
let Ok(serialized) = filtered.marshal_msg() else {
return Rejected;
};
entry.metadata = serialized;
entry.cached = Some(filtered);
}
Resolved(entry)
}
#[derive(Debug, Default)]
pub struct MetaCacheEntries(pub Vec<Option<MetaCacheEntry>>);
@@ -418,11 +618,23 @@ impl MetaCacheEntries {
}
pub fn resolve(&self, params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
self.resolve_inner(params, false)
Self::resolve_inner(&self.0, params, false)
}
pub fn resolve_with_write_quorum(&self, params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
self.resolve_inner(params, true)
Self::resolve_inner(&self.0, params, true)
}
/// Resolve independently observed replicas without copying their metadata.
///
/// Each occupied slot represents one distinct reader. Repeated references
/// are valid only when the caller already established agreement among those
/// readers; pointer equality alone is not evidence of extra replicas.
pub fn resolve_borrowed_with_write_quorum(
entries: &[Option<&MetaCacheEntry>],
params: MetadataResolutionParams,
) -> Option<MetaCacheEntry> {
Self::resolve_inner(entries, params, true)
}
/// Resolve the cross-disk UNION of every version present on ANY disk slot.
@@ -631,8 +843,12 @@ impl MetaCacheEntries {
discovery
}
fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
fn resolve_inner<T: std::borrow::Borrow<MetaCacheEntry>>(
entries: &[Option<T>],
mut params: MetadataResolutionParams,
enforce_write_quorum: bool,
) -> Option<MetaCacheEntry> {
if entries.is_empty() {
debug!(
bucket = %params.bucket,
dir_quorum = params.dir_quorum,
@@ -652,7 +868,7 @@ impl MetaCacheEntries {
let mut objs_agree = 0;
let mut objs_valid = 0;
for (entry_idx, entry) in self.0.iter().flatten().enumerate() {
for (entry_idx, entry) in entries.iter().flatten().map(std::borrow::Borrow::borrow).enumerate() {
debug!(entry = %entry.name, "metacache resolve examining candidate entry");
if entry.name.is_empty() {
continue;
@@ -688,7 +904,9 @@ impl MetaCacheEntries {
};
let same_name = entry.name == current_selected.name;
let outcome = if let Some(fm) = &entry.cached {
let outcome = if std::ptr::eq(entry, *current_selected) {
Some(true)
} else if let Some(fm) = &entry.cached {
let (res, maybe_prev) = selected_borrowed.compare_cached(fm, same_name, params.strict);
if let Some(prev) = maybe_prev {
fallback_borrowed.push((*current_selected_idx, prev));
@@ -753,16 +971,27 @@ impl MetaCacheEntries {
return directory.cloned();
}
if objs_agree == objs_valid {
if objs_agree == objs_valid
&& (!enforce_write_quorum
|| (params.requested_versions == 1
&& selected_borrowed
.versions
.as_slice()
.first()
.is_some_and(|(header, _, _)| !header.free_version())))
{
let required_quorum = if enforce_write_quorum {
selected_borrowed
.versions
.as_slice()
.first()
.map(|(hdr, _, meta)| {
crate::filemeta::write_quorum_from_header_and_meta(hdr, meta, params.obj_quorum)
.saturating_sub(params.write_quorum_slack)
.max(params.obj_quorum)
crate::filemeta::listing_quorum_from_header_and_meta(
hdr,
meta,
params.obj_quorum,
params.write_quorum_slack,
)
})
.unwrap_or(params.obj_quorum)
} else {
@@ -791,7 +1020,7 @@ impl MetaCacheEntries {
let selected_meta_ver = selected_borrowed.meta_ver;
let mut streams: Vec<Vec<(&crate::FileMetaVersionHeader, &[u8])>> = Vec::with_capacity(objs_valid);
for (entry_idx, entry) in self.0.iter().flatten().enumerate() {
for (entry_idx, entry) in entries.iter().flatten().map(std::borrow::Borrow::borrow).enumerate() {
if entry.name.is_empty() || entry.is_dir() || invalid_entries.contains(&entry_idx) {
continue;
}
@@ -816,6 +1045,10 @@ impl MetaCacheEntries {
}
debug_assert_eq!(streams.len(), objs_valid);
let had_user_versions = enforce_write_quorum
&& streams
.iter()
.any(|stream| stream.iter().any(|(header, _)| !header.free_version()));
let versions = crate::filemeta::merge_file_meta_borrowed_streams(
params.obj_quorum,
params.strict,
@@ -824,7 +1057,7 @@ impl MetaCacheEntries {
params.write_quorum_slack,
streams,
);
if versions.is_empty() {
if versions.is_empty() || (had_user_versions && versions.iter().all(|version| version.header.free_version())) {
debug!(
selected = %selected_ref.name,
requested_versions = params.requested_versions,
@@ -921,12 +1154,23 @@ impl MetaCacheEntriesSorted {
entries
}
/// Drop every entry whose name is at or before `marker`.
///
/// Entries are sorted by name, so this keeps the suffix starting at the
/// first name strictly after the marker; when no such entry exists the
/// page is empty. The split index is taken over the raw slots so that
/// `None` placeholders cannot shift it.
pub fn forward_past(&mut self, marker: Option<String>) {
if let Some(val) = marker
&& let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val)
{
self.o.0 = self.o.0.split_off(idx);
}
let Some(val) = marker else {
return;
};
let idx = self
.o
.0
.iter()
.position(|v| v.as_ref().is_some_and(|v| v.name > val))
.unwrap_or(self.o.0.len());
self.o.0 = self.o.0.split_off(idx);
}
}
@@ -1443,6 +1687,58 @@ mod tests {
use tokio::sync::{Notify, oneshot};
use uuid::Uuid;
fn named_entry(name: &str) -> MetaCacheEntry {
MetaCacheEntry {
name: name.to_owned(),
..Default::default()
}
}
#[test]
fn is_in_dir_strips_the_directory_prefix_once() {
assert!(named_entry("a/b").is_in_dir("a/", "/"));
assert!(named_entry("a/b/").is_in_dir("a/", "/"));
assert!(named_entry("a/").is_in_dir("a/", "/"));
// `a/a/b` lives in `a/a/`, not directly in `a/`.
assert!(!named_entry("a/a/b").is_in_dir("a/", "/"));
assert!(!named_entry("b/c").is_in_dir("a/", "/"));
assert!(named_entry("top").is_in_dir("", "/"));
assert!(named_entry("top/").is_in_dir("", "/"));
assert!(!named_entry("top/leaf").is_in_dir("", "/"));
}
#[test]
fn forward_past_drops_every_entry_at_or_before_the_marker() {
let sorted = |names: &[&str]| MetaCacheEntriesSorted {
o: MetaCacheEntries(names.iter().map(|name| Some(named_entry(name))).collect()),
..Default::default()
};
let mut entries = sorted(&["a", "b", "c"]);
entries.forward_past(Some("b".to_owned()));
assert_eq!(entries.entries().iter().map(|e| e.name.as_str()).collect::<Vec<_>>(), ["c"]);
let mut entries = sorted(&["a", "b"]);
entries.forward_past(Some("b".to_owned()));
assert!(entries.entries().is_empty(), "nothing sorts after the marker");
let mut entries = sorted(&["a", "b"]);
entries.forward_past(Some("z".to_owned()));
assert!(entries.entries().is_empty(), "a marker past every entry empties the page");
let mut entries = sorted(&["a", "b"]);
entries.forward_past(None);
assert_eq!(entries.entries().len(), 2, "no marker keeps the page intact");
// A `None` slot before the split point must not shift the retained suffix.
let mut entries = MetaCacheEntriesSorted {
o: MetaCacheEntries(vec![None, Some(named_entry("a")), Some(named_entry("b")), Some(named_entry("c"))]),
..Default::default()
};
entries.forward_past(Some("a".to_owned()));
assert_eq!(entries.entries().iter().map(|e| e.name.as_str()).collect::<Vec<_>>(), ["b", "c"]);
}
#[tokio::test]
async fn test_writer() {
let mut f = Cursor::new(Vec::new());
@@ -2861,6 +3157,171 @@ mod tests {
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
}
#[test]
fn borrowed_resolution_counts_reader_slots_and_preserves_owned_semantics() {
let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time");
for cached in [false, true] {
for legacy in [false, true] {
let mut entry = metacache_entry_with_erasure(time, "etag", 8, 8);
if legacy {
entry = metacache_entry_without_header_ec(entry);
}
if !cached {
entry.cached = None;
}
for replicas in [0, 1, 7, 8, 9] {
let mut slots = vec![Some(&entry); replicas];
slots.extend([None, None]);
let params = MetadataResolutionParams {
dir_quorum: 8,
obj_quorum: 8,
requested_versions: 1,
write_quorum_slack: 8,
bucket: "bucket".to_owned(),
..Default::default()
};
let borrowed = MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone());
let owned = MetaCacheEntries(slots.iter().map(|slot| slot.map(Clone::clone)).collect())
.resolve_with_write_quorum(params);
assert_eq!(borrowed.is_some(), replicas >= 8, "replicas={replicas}, cached={cached}, legacy={legacy}");
assert_eq!(
borrowed.map(|entry| (entry.name, entry.metadata)),
owned.map(|entry| (entry.name, entry.metadata))
);
}
}
}
}
#[test]
fn borrowed_header_agreement_matches_existing_strict_and_relaxed_contracts() {
let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time");
let current = metacache_entry_with_erasure(time, "current", 8, 8);
let older = metacache_entry_with_erasure(time - time::Duration::seconds(1), "older", 8, 8);
let legacy = metacache_entry_without_header_ec(current.clone());
let mut changed_signature = current.clone();
changed_signature.cached.as_mut().expect("cached fixture").versions[0]
.header
.signature[0] ^= 1;
changed_signature.metadata = changed_signature
.cached
.as_ref()
.expect("cached fixture")
.marshal_msg()
.expect("changed header");
let corrupt = MetaCacheEntry {
name: "object".to_owned(),
metadata: vec![0xff],
..Default::default()
};
let directory = metacache_dir_entry("object");
let cases = [current, older, legacy, changed_signature, corrupt, directory];
for strict in [false, true] {
for cached in [false, true] {
for left in &cases {
for right in &cases {
let mut left = left.clone();
let mut right = right.clone();
if !cached {
left.cached = None;
right.cached = None;
}
assert_eq!(
left.headers_agree([&right], strict),
left.matches(Some(&right), strict).1,
"strict={strict}, cached={cached}"
);
}
}
}
}
}
#[test]
fn header_groups_keep_the_original_geometry_baseline_across_legacy_wildcards() {
let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time");
let original = metacache_entry_with_erasure(time, "etag", 8, 8);
let legacy = metacache_entry_without_header_ec(original.clone());
let mut conflict = original.clone();
let metadata = conflict.cached.as_mut().expect("fixture cache");
metadata.versions[0].header.ec_m = 12;
metadata.versions[0].header.ec_n = 4;
conflict.metadata = metadata.marshal_msg().expect("conflicting geometry header");
for strict in [false, true] {
for cache_mask in 0..8 {
let mut replicas = [original.clone(), legacy.clone(), conflict.clone()];
for (index, entry) in replicas.iter_mut().enumerate() {
if cache_mask & (1 << index) == 0 {
entry.cached = None;
}
}
for base in &replicas {
for order in [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] {
let group = order.map(|index| &replicas[index]);
let expected = group.iter().all(|candidate| base.matches(Some(candidate), strict).1);
assert_eq!(
base.headers_agree(group, strict),
expected,
"strict={strict}, mask={cache_mask}, order={order:?}"
);
}
}
}
}
}
#[test]
fn agreed_resolution_preserves_buffers_without_materializing_unchanged_metadata() {
let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time");
for cached in [false, true] {
let mut entry = metacache_entry_with_erasure(time, "etag", 8, 8);
if !cached {
entry.cached = None;
}
let metadata = entry.metadata.as_ptr();
let cached_version = entry.cached.as_ref().map(|meta| meta.versions[0].meta.as_ptr());
let MetaCacheEntryResolution::Resolved(entry) = resolve_agreed_entry(
entry,
8,
&MetadataResolutionParams {
dir_quorum: 8,
obj_quorum: 8,
requested_versions: 1,
write_quorum_slack: 8,
..Default::default()
},
true,
) else {
panic!("eight independent EC8 readers must resolve");
};
assert_eq!(entry.metadata.as_ptr(), metadata, "unchanged serialized metadata must move");
assert_eq!(
entry.cached.as_ref().map(|meta| meta.versions[0].meta.as_ptr()),
cached_version,
"cold metadata stays borrowed and existing decoded buffers move"
);
}
}
#[test]
fn listing_quorum_never_relaxes_a_versions_read_quorum() {
let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time");
for (data, parity, copies, expected) in [(8, 8, 8, true), (12, 4, 8, false), (12, 4, 11, false), (12, 4, 12, true)] {
let entry = metacache_entry_with_erasure(time, "etag", data, parity);
let entries = MetaCacheEntries(vec![Some(entry); copies]);
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 8,
dir_quorum: 8,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
write_quorum_slack: 16 - copies,
..Default::default()
});
assert_eq!(resolved.is_some(), expected, "layout {data}+{parity}, {copies} copies");
}
}
#[test]
fn resolve_with_write_quorum_relaxes_requirement_by_unreachable_drives() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
crates/heal/src/error.rs
+3
View File
@@ -85,6 +85,9 @@ pub enum Error {
#[error("stale_bucket_incarnation: bucket {bucket} no longer belongs to this heal admission ({expected:?})")]
StaleBucketIncarnation { bucket: String, expected: Option<uuid::Uuid> },
#[error("Bucket identity lookup returned a nil incarnation")]
InvalidBucketIncarnation,
/// The current page already exhausted its local retry budget. Retrying
/// the enclosing bucket would replay pages whose results were counted.
#[error("Heal listing failed for bucket {bucket}: {source}")]
crates/heal/src/heal/manager.rs
+15 -5
View File
@@ -1990,15 +1990,25 @@ impl HealManager {
pub(crate) async fn durable_mrf_repair_anchor(
&self,
intent: &rustfs_common::mrf_channel::MrfIntent,
) -> Option<rustfs_common::mrf_channel::MrfDurableRepairAnchor> {
) -> Result<Option<rustfs_common::mrf_channel::MrfDurableRepairAnchor>> {
if let Some(purge) = intent.delete_marker_purge.as_ref() {
return rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, purge.bucket_incarnation_id);
return Ok(rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(
intent,
purge.bucket_incarnation_id,
));
}
match self.storage.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await {
Ok(Some(bucket_incarnation_id)) => {
Ok(match self.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await? {
Some(bucket_incarnation_id) => {
rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, bucket_incarnation_id)
}
Ok(None) | Err(_) => None,
None => None,
})
}
pub(crate) async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
match self.storage.mrf_bucket_incarnation_id(bucket).await? {
Some(incarnation) if incarnation.is_nil() => Err(Error::InvalidBucketIncarnation),
identity => Ok(identity),
}
}
crates/heal/src/heal/mrf_queue.rs
+133 -42
View File
@@ -58,12 +58,18 @@ static GLOBAL_MRF_LIFECYCLE_SENDER: std::sync::OnceLock<mpsc::Sender<MrfLifecycl
pub enum MrfLifecycleControlError {
#[error("MRF lifecycle controller is unavailable")]
Unavailable,
#[error("the MRF checkpoint is unreadable; retain its files and restore a compatible reader")]
CheckpointUnreadable,
#[error("the requested responsibility changed or does not exist")]
StaleGeneration,
#[error("the responsibility is not held as unverified legacy")]
NotHeld,
#[error("the bucket incarnation changed")]
IncarnationChanged,
#[error("the current bucket identity is unresolved; the responsibility remains held")]
BucketIdentityUnresolved,
#[error("the current bucket identity is unavailable; retry after storage recovers")]
BucketIdentityUnavailable(#[source] Box<crate::Error>),
#[error("operator action is invalid: {0}")]
InvalidAction(String),
#[error("the lifecycle checkpoint could not be persisted")]
@@ -115,7 +121,7 @@ pub async fn list_legacy_responsibilities(
.send(MrfLifecycleCommand::List { after, limit, response })
.await
.map_err(|_| MrfLifecycleControlError::Unavailable)?;
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)?
}
pub async fn recheck_legacy_responsibility(
@@ -172,7 +178,7 @@ enum MrfLifecycleCommand {
List {
after: Option<Uuid>,
limit: usize,
response: tokio::sync::oneshot::Sender<MrfLegacyResponsibilitySnapshot>,
response: tokio::sync::oneshot::Sender<Result<MrfLegacyResponsibilitySnapshot, MrfLifecycleControlError>>,
},
Recheck {
responsibility_id: Uuid,
@@ -615,6 +621,16 @@ struct MrfLifecyclePayload {
records: Vec<ResponsibilityCheckpoint>,
}
#[derive(Debug, thiserror::Error)]
enum MrfReplayError {
#[error(transparent)]
Snapshot(#[from] snapshot::SnapshotError),
#[error("MRF lifecycle checkpoint is invalid")]
InvalidLifecycle,
#[error("MRF lifecycle checkpoint could not be decoded")]
Lifecycle(#[source] serde_json::Error),
}
fn encode_mrf_lifecycle_checkpoint(
owner: Uuid,
sequence: u64,
@@ -660,24 +676,25 @@ fn encode_mrf_lifecycle_checkpoint(
Some(encoded)
}
fn decode_mrf_lifecycle_checkpoint(data: &[u8], byte_limit: usize) -> Option<MrfLifecyclePayload> {
fn decode_mrf_lifecycle_checkpoint(data: &[u8], byte_limit: usize) -> Result<MrfLifecyclePayload, MrfReplayError> {
const HEADER_LEN: usize = 8 + 1 + 8 + 32;
if data.len() < HEADER_LEN || data.len() > byte_limit || &data[..8] != MRF_LIFECYCLE_MAGIC {
return None;
return Err(MrfReplayError::InvalidLifecycle);
}
if data[8] != MRF_LIFECYCLE_FORMAT_VERSION {
return None;
return Err(snapshot::SnapshotError::Unsupported.into());
}
let payload_len = usize::try_from(u64::from_le_bytes(data[9..17].try_into().ok()?)).ok()?;
let payload_len = usize::try_from(u64::from_le_bytes(data[9..17].try_into().map_err(|_| MrfReplayError::InvalidLifecycle)?))
.map_err(|_| MrfReplayError::InvalidLifecycle)?;
if payload_len != data.len().saturating_sub(HEADER_LEN) {
return None;
return Err(MrfReplayError::InvalidLifecycle);
}
let payload = &data[HEADER_LEN..];
let digest: [u8; 32] = sha2::Sha256::digest(payload).into();
if digest != data[17..49] {
return None;
return Err(MrfReplayError::InvalidLifecycle);
}
let decoded: MrfLifecyclePayload = serde_json::from_slice(payload).ok()?;
let decoded: MrfLifecyclePayload = serde_json::from_slice(payload).map_err(MrfReplayError::Lifecycle)?;
if decoded.format_version != MRF_LIFECYCLE_FORMAT_VERSION
|| decoded.checkpoint_owner.is_nil()
|| decoded.checkpoint_sequence == 0
@@ -698,9 +715,9 @@ fn decode_mrf_lifecycle_checkpoint(data: &[u8], byte_limit: usize) -> Option<Mrf
.len()
!= decoded.records.len()
{
return None;
return Err(MrfReplayError::InvalidLifecycle);
}
Some(decoded)
Ok(decoded)
}
fn intent_digest(intent: &MrfIntent) -> Option<[u8; 32]> {
@@ -754,8 +771,8 @@ async fn read_mrf_lifecycle_checkpoint(
sequence: u64,
checkpoint_max_bytes: usize,
byte_limit: usize,
) -> Option<Vec<ResponsibilityCheckpoint>> {
let bytes = snapshot::read_committed_companion(
) -> Result<Option<Vec<ResponsibilityCheckpoint>>, MrfReplayError> {
let Some(bytes) = snapshot::read_committed_companion(
&journal_disks().await,
owner,
sequence,
@@ -763,10 +780,15 @@ async fn read_mrf_lifecycle_checkpoint(
checkpoint_max_bytes,
byte_limit,
)
.await
.ok()??;
.await?
else {
return Ok(None);
};
let decoded = decode_mrf_lifecycle_checkpoint(&bytes, byte_limit)?;
(decoded.checkpoint_owner == owner && decoded.checkpoint_sequence == sequence).then_some(decoded.records)
if decoded.checkpoint_owner != owner || decoded.checkpoint_sequence != sequence {
return Err(MrfReplayError::InvalidLifecycle);
}
Ok(Some(decoded.records))
}
/// Write the snapshot to every local disk; returns true when at least one
@@ -944,6 +966,7 @@ struct MrfRuntime {
/// True when replay observed a responsibility that cannot be discharged by
/// a complete verified repair proof in this process.
retain_replay_journal: bool,
replay_readable: bool,
/// Durable responsibilities accepted from replay and waiting for an
/// exact storage-owned proof before the startup journal can be deleted.
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
@@ -999,6 +1022,9 @@ impl MrfRuntime {
intent: MrfIntent,
source_bucket_incarnation_id: Option<Uuid>,
) -> Result<(), MrfDurableAdmissionError> {
if !self.replay_readable {
return Err(MrfDurableAdmissionError::Persistence);
}
self.partial_writes.admit_with_source_incarnation(
intent,
source_bucket_incarnation_id,
@@ -1009,10 +1035,17 @@ impl MrfRuntime {
Ok(())
}
fn list_unverified_legacy(&self, after: Option<Uuid>, limit: usize) -> MrfLegacyResponsibilitySnapshot {
fn list_unverified_legacy(
&self,
after: Option<Uuid>,
limit: usize,
) -> Result<MrfLegacyResponsibilitySnapshot, MrfLifecycleControlError> {
if !self.replay_readable {
return Err(MrfLifecycleControlError::CheckpointUnreadable);
}
let (responsibilities, next_cursor) = self.partial_writes.list_unverified_legacy(after, limit);
let lifecycle = self.partial_writes.lifecycle_summary();
MrfLegacyResponsibilitySnapshot {
Ok(MrfLegacyResponsibilitySnapshot {
contract_version: 2,
node_local: true,
checkpoint_owner: self.runtime_checkpoint.map(|(owner, _)| owner),
@@ -1023,7 +1056,7 @@ impl MrfRuntime {
operator_accepted_count: lifecycle.operator_accepted_count,
responsibilities,
next_cursor,
}
})
}
async fn verify_legacy_generation_incarnation(
@@ -1032,6 +1065,9 @@ impl MrfRuntime {
responsibility_id: Uuid,
expected_bucket_incarnation_id: Uuid,
) -> Result<MrfIntent, MrfLifecycleControlError> {
if !self.replay_readable {
return Err(MrfLifecycleControlError::CheckpointUnreadable);
}
let (intent, state, source_bucket_incarnation_id) = self
.partial_writes
.intent_for_responsibility(responsibility_id)
@@ -1047,7 +1083,8 @@ impl MrfRuntime {
let anchor = manager
.durable_mrf_repair_anchor(&intent)
.await
.ok_or(MrfLifecycleControlError::IncarnationChanged)?;
.map_err(|error| MrfLifecycleControlError::BucketIdentityUnavailable(Box::new(error)))?
.ok_or(MrfLifecycleControlError::BucketIdentityUnresolved)?;
if anchor.bucket_incarnation_id != expected_bucket_incarnation_id {
return Err(MrfLifecycleControlError::IncarnationChanged);
}
@@ -1060,6 +1097,9 @@ impl MrfRuntime {
responsibility_id: Uuid,
expected_bucket_incarnation_id: Uuid,
) -> Result<(), MrfLifecycleControlError> {
if !self.replay_readable {
return Err(MrfLifecycleControlError::CheckpointUnreadable);
}
if self.partial_writes.is_active_responsibility(responsibility_id) {
return Ok(());
}
@@ -1088,6 +1128,9 @@ impl MrfRuntime {
responsibility_id: Uuid,
expected_bucket_incarnation_id: Option<Uuid>,
) -> Result<(), MrfLifecycleControlError> {
if !self.replay_readable {
return Err(MrfLifecycleControlError::CheckpointUnreadable);
}
let (intent, state, source_incarnation) = self
.partial_writes
.intent_for_responsibility(responsibility_id)
@@ -1096,14 +1139,15 @@ impl MrfRuntime {
return Err(MrfLifecycleControlError::StaleGeneration);
}
if expected_bucket_incarnation_id.is_none()
&& (source_incarnation.is_some() || !matches!(state, partial_write::ResponsibilityState::Active))
&& (source_incarnation.is_some() || !matches!(state, partial_write::ResponsibilityState::Active {}))
{
return Err(MrfLifecycleControlError::StaleGeneration);
}
let anchor = manager
.durable_mrf_repair_anchor(&intent)
.await
.ok_or(MrfLifecycleControlError::IncarnationChanged)?;
.map_err(|error| MrfLifecycleControlError::BucketIdentityUnavailable(Box::new(error)))?
.ok_or(MrfLifecycleControlError::BucketIdentityUnresolved)?;
if Some(anchor.bucket_incarnation_id) == expected_bucket_incarnation_id {
return Ok(());
}
@@ -1133,6 +1177,9 @@ impl MrfRuntime {
manager: &HealManager,
request: MrfLegacyRiskAcceptanceRequest,
) -> Result<(), MrfLifecycleControlError> {
if !self.replay_readable {
return Err(MrfLifecycleControlError::CheckpointUnreadable);
}
let responsibility_id = request.responsibility_id;
if self
.partial_writes
@@ -1186,6 +1233,11 @@ impl MrfRuntime {
}
async fn flush(&mut self) -> bool {
// A new snapshot or compatibility mirror must never replace an owner
// whose lifecycle records this process could not read completely.
if !self.replay_readable {
return false;
}
let (authoritative, legacy) = self.snapshot();
let lifecycle_byte_limit = self.config.journal_max_bytes.saturating_mul(4);
let lifecycle_checkpoint = if self.partial_writes.depth() == 0 {
@@ -1286,6 +1338,9 @@ impl MrfRuntime {
}
async fn dispatch(&mut self, manager: &HealManager) {
if !self.replay_readable {
return;
}
self.dispatch_partial_writes(manager).await;
if let Some(until) = self.backoff_until {
if tokio::time::Instant::now() < until {
@@ -1357,6 +1412,9 @@ impl MrfRuntime {
}
async fn delete_idle_recovery_anchors(&mut self) -> bool {
if !self.replay_readable {
return false;
}
let runtime_deleted = match self.runtime_checkpoint {
Some((owner, sequence)) => {
match snapshot::delete_committed_snapshots_through(owner, sequence, self.config.journal_max_bytes).await {
@@ -1522,6 +1580,7 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
dirty: false,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
@@ -1537,6 +1596,7 @@ struct ReplayOutcome {
replayed: usize,
journal_on_disk: bool,
retain_journal_for_replay: bool,
source_readable: bool,
durable_replay_anchors: Vec<MrfDurableRepairAnchor>,
partial_writes: Vec<(MrfIntent, Option<ResponsibilityCheckpoint>)>,
cleanup: Option<ReplayCleanup>,
@@ -1559,7 +1619,8 @@ enum ReplayCleanup {
}
struct ReplaySource {
data: Vec<u8>,
intents: Vec<MrfIntent>,
truncated_bytes: usize,
cleanup: ReplayCleanup,
lifecycle: HashMap<[u8; 32], VecDeque<ResponsibilityCheckpoint>>,
}
@@ -1576,22 +1637,38 @@ fn take_lifecycle_record(
record
}
async fn read_replay_source(
max_bytes: usize,
lifecycle_max_bytes: usize,
) -> Result<Option<ReplaySource>, snapshot::SnapshotError> {
async fn read_replay_source(max_bytes: usize, lifecycle_max_bytes: usize) -> Result<Option<ReplaySource>, MrfReplayError> {
if let Some(committed) = snapshot::inspect_local_committed_snapshot(max_bytes).await? {
let owner = committed.owner();
let sequence = committed.sequence();
let records = read_mrf_lifecycle_checkpoint(owner, sequence, max_bytes, lifecycle_max_bytes)
.await
.unwrap_or_default();
let (intents, truncated_bytes) = decode_journal(committed.payload());
let records = read_mrf_lifecycle_checkpoint(owner, sequence, max_bytes, lifecycle_max_bytes).await?;
if let Some(records) = records.as_ref() {
// Validate the complete owner before rearming or submitting any
// intent. Different source generations may share an intent digest.
let mut counts = HashMap::new();
for intent in intents.iter().filter(|intent| intent.kind.is_durable()) {
let digest = intent_digest(intent).ok_or(MrfReplayError::InvalidLifecycle)?;
*counts.entry(digest).or_insert(0usize) += 1;
}
for record in records {
match counts.get_mut(&record.intent_digest) {
Some(count) if *count > 0 => *count -= 1,
_ => return Err(MrfReplayError::InvalidLifecycle),
}
}
if counts.values().any(|count| *count != 0) {
return Err(MrfReplayError::InvalidLifecycle);
}
}
let records = records.unwrap_or_default();
let mut lifecycle: HashMap<[u8; 32], VecDeque<ResponsibilityCheckpoint>> = HashMap::with_capacity(records.len());
for record in records {
lifecycle.entry(record.intent_digest).or_default().push_back(record);
}
return Ok(Some(ReplaySource {
data: committed.payload().to_vec(),
intents,
truncated_bytes,
cleanup: ReplayCleanup::Committed { owner, sequence },
lifecycle,
}));
@@ -1606,8 +1683,10 @@ async fn read_replay_source(
None => return Ok(None),
},
};
let (intents, truncated_bytes) = decode_journal(&data);
Ok(Some(ReplaySource {
data,
intents,
truncated_bytes,
cleanup: ReplayCleanup::Legacy,
lifecycle: HashMap::new(),
}))
@@ -1649,6 +1728,7 @@ async fn replay_into(
replayed: 0,
journal_on_disk: false,
retain_journal_for_replay: false,
source_readable: true,
durable_replay_anchors: Vec::new(),
partial_writes: Vec::new(),
cleanup: None,
@@ -1665,6 +1745,7 @@ async fn replay_into(
replayed: 0,
journal_on_disk: true,
retain_journal_for_replay: true,
source_readable: false,
durable_replay_anchors: Vec::new(),
partial_writes: Vec::new(),
cleanup: None,
@@ -1677,11 +1758,10 @@ async fn replay_into(
ReplayCleanup::Legacy => 1,
ReplayCleanup::Committed { sequence, .. } => sequence.saturating_add(1),
};
let data = source.data;
let mut lifecycle = source.lifecycle;
let (decoded, truncated) = decode_journal(&data);
let replayed = decoded.len();
let intents = decoded;
let intents = source.intents;
let truncated = source.truncated_bytes;
let replayed = intents.len();
if truncated > 0 {
tracing::warn!(
target: "rustfs::heal::mrf",
@@ -1712,7 +1792,7 @@ async fn replay_into(
rustfs_common::mrf_channel::release_mrf_intent(&intent);
continue;
}
if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&intent).await {
durable_replay_anchors.push(anchor);
}
let restored = intent_digest(&intent).and_then(|digest| take_lifecycle_record(&mut lifecycle, digest));
@@ -1750,7 +1830,7 @@ async fn replay_into(
// Adopt durable replay into its single checkpointed owner. The
// runtime dispatches it after publishing the successor, avoiding
// a duplicate task in the startup-replay and steady-state paths.
if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&intent).await {
durable_replay_anchors.push(anchor.clone());
}
let restored = intent_digest(&intent).and_then(|digest| take_lifecycle_record(&mut lifecycle, digest));
@@ -1759,7 +1839,7 @@ async fn replay_into(
}
match submit_mrf_heal_request(manager, &intent, None).await {
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {
if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await {
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&intent).await {
durable_replay_anchors.push(anchor);
} else {
accepted_without_durable_anchor = true;
@@ -1811,6 +1891,7 @@ async fn replay_into(
replayed,
journal_on_disk,
retain_journal_for_replay,
source_readable: true,
durable_replay_anchors,
partial_writes,
cleanup: journal_on_disk.then_some(cleanup),
@@ -1837,6 +1918,7 @@ async fn run_mrf_consumer(
dirty: false,
journal_on_disk: false,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
@@ -1848,6 +1930,7 @@ async fn run_mrf_consumer(
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
runtime.journal_on_disk = replay.journal_on_disk;
runtime.retain_replay_journal = replay.retain_journal_for_replay;
runtime.replay_readable = replay.source_readable;
runtime.durable_replay_anchors = replay.durable_replay_anchors;
runtime.replay_cleanup = replay.cleanup;
runtime.next_checkpoint_sequence = replay.next_checkpoint_sequence;
@@ -2064,6 +2147,7 @@ mod tests {
use std::time::{Duration as StdDuration, Instant};
mod empty_events_tests;
mod lifecycle_replay_tests;
#[tokio::test(start_paused = true)]
async fn journal_disk_io_bounds_fanout_and_collects_every_result() {
@@ -2232,8 +2316,8 @@ mod tests {
let mut torn = encoded.clone();
*torn.last_mut().expect("non-empty checkpoint") ^= 0xff;
assert!(decode_mrf_lifecycle_checkpoint(&torn, 8192).is_none());
assert!(decode_mrf_lifecycle_checkpoint(&encoded, encoded.len() - 1).is_none());
assert!(decode_mrf_lifecycle_checkpoint(&torn, 8192).is_err());
assert!(decode_mrf_lifecycle_checkpoint(&encoded, encoded.len() - 1).is_err());
assert!(encode_mrf_lifecycle_checkpoint(Uuid::nil(), 42, Vec::new(), 8192).is_none());
}
@@ -2362,6 +2446,7 @@ mod tests {
dirty: false,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
@@ -2501,6 +2586,7 @@ mod tests {
dirty: true,
journal_on_disk: false,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
@@ -3093,6 +3179,7 @@ mod tests {
dirty: false,
journal_on_disk: true,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: vec![anchor],
replay_cleanup: Some(cleanup),
runtime_checkpoint: None,
@@ -3144,6 +3231,7 @@ mod tests {
dirty: false,
journal_on_disk: true,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
@@ -3192,6 +3280,7 @@ mod tests {
dirty: false,
journal_on_disk: true,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: Some(ReplayCleanup::Committed {
owner: replay_owner,
@@ -3306,7 +3395,7 @@ mod tests {
responsibility_id: Uuid::new_v4(),
source_bucket_incarnation_id: Some(source_bucket_incarnation_id),
last_operator_acceptance: None,
state: partial_write::ResponsibilityState::Active,
state: partial_write::ResponsibilityState::Active {},
}],
lifecycle_limit,
)
@@ -3366,6 +3455,7 @@ mod tests {
dirty: false,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
@@ -3613,6 +3703,7 @@ mod tests {
dirty: true,
journal_on_disk: false,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
crates/heal/src/heal/mrf_queue/partial_write.rs
+115 -26
View File
@@ -34,6 +34,7 @@ const MAX_OPERATOR_REFERENCE_BYTES: usize = 256;
const LIFECYCLE_CHECKPOINT_RECORD_OVERHEAD_BYTES: usize = 256;
const MAX_UNVERIFIED_RETRY_BACKOFF: Duration = Duration::from_secs(60);
const MRF_RETIREMENT_PROOF_RECHECK_INTERVAL: Duration = Duration::from_secs(5);
const MRF_IDENTITY_PROBE_TIMEOUT: Duration = Duration::from_secs(1);
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct PartialWriteKey {
@@ -67,7 +68,7 @@ impl PartialWriteKey {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
pub(super) enum ResponsibilityState {
Active,
Active {},
HeldUnverifiedLegacy {
bucket_incarnation_id: Uuid,
since_ms: u64,
@@ -95,12 +96,12 @@ pub(super) enum ResponsibilityState {
impl ResponsibilityState {
fn is_parked(&self) -> bool {
!matches!(self, Self::Active)
!matches!(self, Self::Active {})
}
fn estimated_bytes(&self) -> usize {
match self {
Self::Active => 0,
Self::Active {} => 0,
Self::HeldUnverifiedLegacy { .. } => std::mem::size_of::<Uuid>() + std::mem::size_of::<u64>(),
Self::LegacyGenerationUnknown { .. } => std::mem::size_of::<Uuid>() + std::mem::size_of::<u64>(),
Self::BucketIncarnationChanged { .. } => 2 * std::mem::size_of::<Uuid>() + std::mem::size_of::<u64>(),
@@ -115,7 +116,7 @@ impl ResponsibilityState {
pub(super) fn bucket_incarnation_id(&self) -> Option<Uuid> {
match self {
Self::Active => None,
Self::Active {} => None,
Self::HeldUnverifiedLegacy {
bucket_incarnation_id, ..
}
@@ -156,7 +157,7 @@ impl ResponsibilityCheckpoint {
return false;
}
match &self.state {
ResponsibilityState::Active => true,
ResponsibilityState::Active {} => true,
ResponsibilityState::HeldUnverifiedLegacy {
bucket_incarnation_id, ..
} => !bucket_incarnation_id.is_nil() && self.source_bucket_incarnation_id == Some(*bucket_incarnation_id),
@@ -207,11 +208,13 @@ struct Responsibility {
// Process-local pacing does not alter the durable proof obligation.
attempt_in_flight: bool,
consecutive_unverified_attempts: u32,
last_identity_probe_result: Option<&'static str>,
last_identity_probe_at_ms: Option<u64>,
}
impl Responsibility {
fn generation_unobserved(&self) -> bool {
self.source_bucket_incarnation_id.is_none() && matches!(self.state, ResponsibilityState::Active)
self.source_bucket_incarnation_id.is_none() && matches!(self.state, ResponsibilityState::Active {})
}
fn is_parked(&self) -> bool {
@@ -255,21 +258,24 @@ pub(super) struct PartialWrites {
entries: HashMap<PartialWriteKey, Responsibility>,
retry_order: VecDeque<PartialWriteKey>,
retry_index: HashSet<PartialWriteKey>,
identity_probe_order: VecDeque<PartialWriteKey>,
identity_probe_index: HashSet<PartialWriteKey>,
retirement_checks: HashMap<(Arc<str>, Uuid), (Instant, bool)>,
bytes: usize,
}
impl PartialWrites {
pub(super) fn cost(intent: &MrfIntent) -> usize {
Self::cost_with_state(intent, &ResponsibilityState::Active)
Self::cost_with_state(intent, &ResponsibilityState::Active {})
}
pub(super) fn cost_with_state(intent: &MrfIntent, state: &ResponsibilityState) -> usize {
intent.estimated_bytes()
+ std::mem::size_of::<Responsibility>()
+ 2 * std::mem::size_of::<PartialWriteKey>()
+ 4 * std::mem::size_of::<PartialWriteKey>()
+ LIFECYCLE_CHECKPOINT_RECORD_OVERHEAD_BYTES
+ state.estimated_bytes()
// Reserve the observation transition before admitting an intent.
+ state.estimated_bytes().max(std::mem::size_of::<Uuid>() + std::mem::size_of::<u64>())
}
pub(super) fn cost_with_state_and_audit(
@@ -326,6 +332,9 @@ impl PartialWrites {
}
retry_queued = true;
}
if source_bucket_incarnation_id.is_none() && self.identity_probe_index.insert(key.clone()) {
self.identity_probe_order.push_back(key.clone());
}
self.entries.insert(
key,
Responsibility {
@@ -335,11 +344,18 @@ impl PartialWrites {
responsibility_id: Uuid::new_v4(),
source_bucket_incarnation_id,
last_operator_acceptance: None,
state: ResponsibilityState::Active,
state: ResponsibilityState::Active {},
retry_queued,
next_attempt: Instant::now(),
next_attempt: Instant::now()
+ if source_bucket_incarnation_id.is_none() {
MAX_UNVERIFIED_RETRY_BACKOFF
} else {
Duration::ZERO
},
attempt_in_flight: false,
consecutive_unverified_attempts: 0,
last_identity_probe_result: None,
last_identity_probe_at_ms: None,
},
);
self.bytes = next_bytes;
@@ -450,7 +466,7 @@ impl PartialWrites {
let mut summary = PartialWriteLifecycleSummary::default();
for entry in self.entries.values() {
match &entry.state {
ResponsibilityState::Active => {
ResponsibilityState::Active {} => {
if entry.generation_unobserved() {
summary.legacy_generation_unknown_count += 1;
}
@@ -501,7 +517,7 @@ impl PartialWrites {
let Some(entry) = self.entries.get_mut(&key) else {
return false;
};
if !matches!(&entry.state, ResponsibilityState::Active) {
if !matches!(&entry.state, ResponsibilityState::Active {}) {
return false;
}
let old_cost = Self::entry_cost(entry);
@@ -599,7 +615,7 @@ impl PartialWrites {
ResponsibilityState::OperatorAcceptedUnverified { .. } => {
return Err("a different operator disposition already exists");
}
ResponsibilityState::Active => return Err("responsibility is not held as unverified legacy"),
ResponsibilityState::Active {} => return Err("responsibility is not held as unverified legacy"),
};
if current_incarnation != expected_bucket_incarnation_id {
return Err("bucket incarnation changed; refresh the responsibility listing");
@@ -677,7 +693,7 @@ impl PartialWrites {
}
let previous = entry.state.clone();
let old_cost = Self::entry_cost(entry);
entry.state = ResponsibilityState::Active;
entry.state = ResponsibilityState::Active {};
self.bytes = self.bytes.saturating_sub(old_cost).saturating_add(Self::entry_cost(entry));
entry.persisted = false;
entry.next_attempt = Instant::now();
@@ -725,8 +741,8 @@ impl PartialWrites {
let mut selected = BTreeMap::new();
for entry in self.entries.values() {
let (status, bucket_incarnation_id, held_since_ms) = match &entry.state {
ResponsibilityState::Active if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
ResponsibilityState::Active => continue,
ResponsibilityState::Active {} if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
ResponsibilityState::Active {} => continue,
ResponsibilityState::HeldUnverifiedLegacy {
bucket_incarnation_id,
since_ms,
@@ -772,6 +788,14 @@ impl PartialWrites {
status,
held_since_ms,
accepted: entry.last_operator_acceptance.clone(),
last_identity_probe_result: entry.last_identity_probe_result,
last_identity_probe_at_ms: entry.last_identity_probe_at_ms,
next_identity_probe_at_ms: entry.generation_unobserved().then(|| {
unix_now_ms().saturating_add(
u64::try_from(entry.next_attempt.saturating_duration_since(Instant::now()).as_millis())
.unwrap_or(u64::MAX),
)
}),
};
selected.insert(entry.responsibility_id, row);
if selected.len() > limit + 1 {
@@ -849,10 +873,10 @@ impl PartialWrites {
config: &MrfConsumerConfig,
) -> bool {
let now = Instant::now();
let mut lifecycle_changed = false;
let mut lifecycle_changed = self.probe_unknown_generation(manager, config).await;
let ready = self.ready_keys(now, config.replay_batch);
if ready.is_empty() {
return false;
return lifecycle_changed;
}
let in_flight = manager.try_in_flight_durable_mrf_anchors();
for key in ready {
@@ -873,7 +897,7 @@ impl PartialWrites {
}
// The bucket can be deleted and recreated while an obligation is
// parked, so every dispatch must compare against a fresh identity.
if let Some(anchor) = manager.durable_mrf_repair_anchor(&entry.intent).await {
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&entry.intent).await {
entry.anchor = Some(anchor.clone());
if let Some(source_bucket_incarnation_id) = entry.source_bucket_incarnation_id
&& source_bucket_incarnation_id != anchor.bucket_incarnation_id
@@ -909,6 +933,57 @@ impl PartialWrites {
lifecycle_changed
}
async fn probe_unknown_generation(&mut self, manager: &HealManager, config: &MrfConsumerConfig) -> bool {
if config.replay_batch == 0 {
return false;
}
let now = Instant::now();
// At most one bounded identity lookup per dispatch round. These keys
// never enter the object-heal retry path or acquire a repair anchor.
for _ in 0..self.identity_probe_order.len().min(config.replay_batch) {
let Some(key) = self.identity_probe_order.pop_front() else {
break;
};
self.identity_probe_index.remove(&key);
let Some(entry) = self.entries.get_mut(&key) else {
continue;
};
if !entry.generation_unobserved() {
continue;
}
self.identity_probe_index.insert(key.clone());
self.identity_probe_order.push_back(key);
if !entry.persisted || entry.next_attempt > now {
continue;
}
// Advance before awaiting so cancellation cannot cause a tight
// retry loop. Restart begins with the same conservative delay.
entry.defer_unverified_attempt(now, config.admission_backoff.max(MRF_RETIREMENT_PROOF_RECHECK_INTERVAL));
let result = tokio::time::timeout(
MRF_IDENTITY_PROBE_TIMEOUT,
manager.mrf_bucket_incarnation_id(entry.intent.bucket.as_ref()),
)
.await;
entry.last_identity_probe_at_ms = Some(unix_now_ms());
match result {
Ok(Ok(Some(observed_bucket_incarnation_id))) => {
let old_cost = Self::entry_cost(entry);
entry.state = ResponsibilityState::LegacyGenerationUnknown {
observed_bucket_incarnation_id,
detected_at_ms: unix_now_ms(),
};
entry.last_identity_probe_result = Some("known");
self.bytes = self.bytes.saturating_sub(old_cost).saturating_add(Self::entry_cost(entry));
return true;
}
Ok(Ok(None)) => entry.last_identity_probe_result = Some("unresolved"),
Ok(Err(_)) | Err(_) => entry.last_identity_probe_result = Some("unavailable"),
}
return false;
}
false
}
fn ready_keys(&mut self, now: Instant, limit: usize) -> Vec<PartialWriteKey> {
let mut ready = Vec::with_capacity(limit.min(self.entries.len()));
// Rotation prevents a permanently failing prefix from starving the
@@ -956,6 +1031,8 @@ impl PartialWrites {
self.bytes = self.entries.values().map(Self::entry_cost).sum();
self.retry_order.retain(|key| self.entries.contains_key(key));
self.retry_index.retain(|key| self.entries.contains_key(key));
self.identity_probe_order.retain(|key| self.entries.contains_key(key));
self.identity_probe_index.retain(|key| self.entries.contains_key(key));
true
}
}
@@ -988,6 +1065,13 @@ pub struct MrfLegacyResponsibility {
pub status: &'static str,
pub held_since_ms: Option<u64>,
pub accepted: Option<MrfOperatorAcceptance>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_identity_probe_result: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_identity_probe_at_ms: Option<u64>,
/// Process-local schedule; replay restarts the conservative probe delay.
#[serde(skip_serializing_if = "Option::is_none")]
pub next_identity_probe_at_ms: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -1021,6 +1105,7 @@ mod tests {
use uuid::Uuid;
mod retirement_tests;
mod unknown_source_tests;
fn intent(object: &str) -> MrfIntent {
let mut intent = MrfIntent {
@@ -1148,7 +1233,7 @@ mod tests {
fn lifecycle_summary_counts_mixed_states_without_changing_responsibilities() {
let incarnation = Uuid::new_v4();
let writes = lifecycle_fixture(vec![
ResponsibilityState::Active,
ResponsibilityState::Active {},
ResponsibilityState::HeldUnverifiedLegacy {
bucket_incarnation_id: incarnation,
since_ms: u64::MAX,
@@ -1228,7 +1313,7 @@ mod tests {
assert!(writes.retry_order.is_empty(), "unobserved legacy work must leave the hot retry queue");
let record = writes.checkpoint_records(|_| Some([7; 32])).remove(0);
assert!(record.is_valid());
assert!(matches!(record.state, ResponsibilityState::Active), "keep the legacy disk encoding");
assert!(matches!(record.state, ResponsibilityState::Active {}), "keep the legacy disk encoding");
assert_eq!(record.source_bucket_incarnation_id, None);
let (rows, next) = writes.list_unverified_legacy(None, 1);
assert_eq!(rows.len(), 1);
@@ -2009,7 +2094,7 @@ mod tests {
.expect("current responsibility should be resident");
let current_id = current_entry.responsibility_id;
assert_ne!(old_id, current_id);
assert!(matches!(current_entry.state, ResponsibilityState::Active));
assert!(matches!(current_entry.state, ResponsibilityState::Active {}));
assert!(current_entry.last_operator_acceptance.is_none());
assert!(
!writes.park_unverified_legacy(&old_anchor),
@@ -2017,7 +2102,7 @@ mod tests {
);
assert!(matches!(
writes.intent_for_responsibility(current_id),
Some((_, ResponsibilityState::Active, Some(source))) if source == current_incarnation
Some((_, ResponsibilityState::Active {}, Some(source))) if source == current_incarnation
));
let mut journal = Vec::new();
@@ -2071,6 +2156,7 @@ mod tests {
dirty: false,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
@@ -2085,7 +2171,7 @@ mod tests {
));
assert!(matches!(
runtime.partial_writes.intent_for_responsibility(current_id),
Some((_, ResponsibilityState::Active, Some(source))) if source == current_incarnation
Some((_, ResponsibilityState::Active {}, Some(source))) if source == current_incarnation
));
assert!(runtime.partial_writes.last_operator_acceptance(current_id).is_none());
@@ -2178,7 +2264,7 @@ mod tests {
assert_eq!(writes.lifecycle_summary().legacy_generation_unknown_count, 1);
let original_unknown = writes.checkpoint_records(super::super::intent_digest).remove(0);
assert_eq!(original_unknown.source_bucket_incarnation_id, None);
assert!(matches!(original_unknown.state, ResponsibilityState::Active));
assert!(matches!(original_unknown.state, ResponsibilityState::Active {}));
assert_eq!(writes.list_unverified_legacy(None, 10).0[0].status, "legacy_generation_unobserved");
let current = intent("same-object-unknown-source");
@@ -2232,6 +2318,7 @@ mod tests {
dirty: false,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
@@ -2341,6 +2428,7 @@ mod tests {
dirty: false,
journal_on_disk: true,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
@@ -2417,6 +2505,7 @@ mod tests {
dirty: false,
journal_on_disk: true,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: vec![anchor],
replay_cleanup: None,
runtime_checkpoint: None,
crates/heal/src/heal/mrf_queue/partial_write/tests/unknown_source_tests.rs
+370
View File
@@ -0,0 +1,370 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::heal::mrf_queue::{MrfLifecycleControlError, MrfQueue, MrfRuntime};
use crate::heal::storage::{HealListItem, HealObjectInfo};
use crate::heal::storage_api::storage::{BucketInfo, DiskSetSelector};
use crate::heal::{DiskStore, EcstoreError};
use crate::{Error, Result};
use rustfs_heal_contracts::heal_channel::HealOpts;
use rustfs_madmin::heal_commands::HealResultItem;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone, Copy)]
enum Lookup {
Identity(Option<Uuid>),
PermissionDenied,
Quorum,
Transport,
Pending,
}
struct ProbeStorage {
lookup: Mutex<Lookup>,
calls: AtomicUsize,
}
#[async_trait::async_trait]
impl HealStorageAPI for ProbeStorage {
async fn mrf_bucket_incarnation_id(&self, _: &str) -> Result<Option<Uuid>> {
self.calls.fetch_add(1, Ordering::Relaxed);
let lookup = *self.lookup.lock().expect("lookup outcome");
match lookup {
Lookup::Identity(value) => Ok(value),
Lookup::PermissionDenied => Err(Error::Io(std::io::ErrorKind::PermissionDenied.into())),
Lookup::Quorum => Err(Error::Storage(EcstoreError::ErasureReadQuorum)),
Lookup::Transport => Err(Error::Io(std::io::ErrorKind::ConnectionReset.into())),
Lookup::Pending => std::future::pending().await,
}
}
async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
panic!("identity probing must not read objects")
}
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
panic!("identity probing must not rebuild objects")
}
async fn get_bucket_info(&self, _: &str) -> Result<Option<BucketInfo>> {
panic!("identity probing must use the incarnation lookup")
}
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
panic!("identity probing must not enumerate buckets")
}
async fn list_buckets_for_admin_heal(&self, _: Option<DiskSetSelector>) -> Result<Vec<BucketInfo>> {
panic!("identity probing must not start admin healing")
}
async fn list_objects_for_heal_page(
&self,
_: &str,
_: &str,
_: Option<&str>,
_: bool,
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
panic!("identity probing must not enumerate objects")
}
async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
panic!("identity probing must not probe objects")
}
async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
panic!("identity probing must not heal objects")
}
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
panic!("identity probing must not heal buckets")
}
async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
panic!("identity probing must not heal formats")
}
async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
panic!("identity probing must not open resume disks")
}
}
fn runtime() -> MrfRuntime {
let config = MrfConsumerConfig::default();
MrfRuntime {
partial_writes: PartialWrites::default(),
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
config,
checkpoint_owner: Uuid::new_v4(),
next_checkpoint_sequence: 1,
new_since_flush: 0,
dirty: false,
journal_on_disk: false,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
backoff_until: None,
}
}
#[tokio::test(start_paused = true)]
async fn unknown_source_deleted_bucket_probes_preserve_errors_and_bound_retries() {
let storage = Arc::new(ProbeStorage {
lookup: Mutex::new(Lookup::Identity(None)),
calls: AtomicUsize::new(0),
});
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
let mut runtime = runtime();
let original = intent("identity-probe");
let key = PartialWriteKey::new(&original, None);
runtime
.admit_partial_write(original, None)
.expect("admit a retained obligation");
runtime.partial_writes.mark_persisted();
let checkpoint = runtime.partial_writes.checkpoint_records(super::super::super::intent_digest);
let id = checkpoint[0].responsibility_id;
let bytes = runtime.partial_writes.bytes();
for outcome in [
Lookup::Identity(None),
Lookup::PermissionDenied,
Lookup::Quorum,
Lookup::Transport,
Lookup::Identity(Some(Uuid::nil())),
Lookup::Pending,
] {
*storage.lookup.lock().expect("set lookup outcome") = outcome;
runtime
.partial_writes
.entries
.get_mut(&key)
.expect("retained obligation")
.next_attempt = Instant::now();
let calls = storage.calls.load(Ordering::Relaxed);
let start = Instant::now();
assert!(
!runtime
.partial_writes
.dispatch_collecting_lifecycle_change(&manager, &runtime.config)
.await
);
assert_eq!(storage.calls.load(Ordering::Relaxed), calls + 1);
assert!(start.elapsed() <= MRF_IDENTITY_PROBE_TIMEOUT);
let entry = runtime.partial_writes.entries.get(&key).expect("probe must retain its owner");
assert!(entry.next_attempt > Instant::now());
assert!(entry.next_attempt <= Instant::now() + MAX_UNVERIFIED_RETRY_BACKOFF);
assert!(entry.anchor.is_none(), "lookup failures cannot create a repair proof");
assert_eq!(runtime.partial_writes.bytes(), bytes);
assert_eq!(runtime.partial_writes.checkpoint_records(super::super::super::intent_digest), checkpoint);
assert!(
!runtime
.partial_writes
.dispatch_collecting_lifecycle_change(&manager, &runtime.config)
.await
);
assert_eq!(
storage.calls.load(Ordering::Relaxed),
calls + 1,
"a consumer tick cannot bypass probe backoff"
);
let row = runtime
.list_unverified_legacy(None, 1)
.expect("list retained obligation")
.responsibilities
.remove(0);
assert_eq!(row.bucket_incarnation_id, None);
assert_eq!(row.source_bucket_incarnation_id, None);
assert_eq!(
row.last_identity_probe_result,
Some(if matches!(outcome, Lookup::Identity(None)) {
"unresolved"
} else {
"unavailable"
})
);
assert!(row.next_identity_probe_at_ms.is_some());
if !matches!(outcome, Lookup::Pending) {
let error = runtime
.refresh_legacy_generation(&manager, id, None)
.await
.expect_err("refresh needs a known identity");
match (outcome, error) {
(Lookup::Identity(None), MrfLifecycleControlError::BucketIdentityUnresolved) => {}
(Lookup::PermissionDenied, MrfLifecycleControlError::BucketIdentityUnavailable(source)) => {
assert!(matches!(*source, Error::Io(ref error) if error.kind() == std::io::ErrorKind::PermissionDenied));
}
(Lookup::Quorum, MrfLifecycleControlError::BucketIdentityUnavailable(source)) => {
assert!(matches!(*source, Error::Storage(EcstoreError::ErasureReadQuorum)));
}
(Lookup::Transport, MrfLifecycleControlError::BucketIdentityUnavailable(source)) => {
assert!(matches!(*source, Error::Io(ref error) if error.kind() == std::io::ErrorKind::ConnectionReset));
}
(Lookup::Identity(Some(_)), MrfLifecycleControlError::BucketIdentityUnavailable(source)) => {
assert!(matches!(*source, Error::InvalidBucketIncarnation));
}
(_, error) => panic!("lookup error lost its type: {error:?}"),
}
}
}
assert_eq!(manager.get_queue_length().await, 0);
}
#[tokio::test(start_paused = true)]
async fn unknown_source_deleted_bucket_probe_rotation_keeps_count_and_byte_budgets() {
let current = Uuid::new_v4();
let storage = Arc::new(ProbeStorage {
lookup: Mutex::new(Lookup::Identity(Some(current))),
calls: AtomicUsize::new(0),
});
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
let mut writes = PartialWrites::default();
let config = MrfConsumerConfig {
replay_batch: 1,
..Default::default()
};
for object in ["rotation-a", "rotation-b", "rotation-c"] {
let item = intent(object);
let budget = writes.bytes() + PartialWrites::cost(&item);
writes.admit(item, 3, budget).expect("admit at the exact byte boundary");
}
let bytes = writes.bytes();
assert!(writes.admit(intent("overflow"), 3, bytes).is_err());
writes.mark_persisted();
for entry in writes.entries.values_mut() {
entry.next_attempt = Instant::now();
}
for expected_calls in 1..=3 {
assert!(writes.dispatch_collecting_lifecycle_change(&manager, &config).await);
assert_eq!(storage.calls.load(Ordering::Relaxed), expected_calls);
assert_eq!(writes.bytes(), bytes, "observation must fit the budget reserved at admission");
}
let (rows, _) = writes.list_unverified_legacy(None, 10);
assert_eq!(rows.len(), 3);
assert!(
rows.iter()
.all(|row| row.source_bucket_incarnation_id.is_none() && row.bucket_incarnation_id == Some(current))
);
for _ in 0..3 {
assert!(!writes.dispatch_collecting_lifecycle_change(&manager, &config).await);
}
assert_eq!(storage.calls.load(Ordering::Relaxed), 3, "observed unknown sources stay parked");
assert_eq!(manager.get_queue_length().await, 0);
}
#[test]
#[serial]
fn unknown_source_deleted_bucket_replay_probes_recreation_without_rebinding() {
// Match the bounded stack used by other deep real-storage heal fixtures.
const STACK_SIZE: usize = 8 * 1024 * 1024;
std::thread::Builder::new()
.name("mrf-unknown-source-replay".to_owned())
.stack_size(STACK_SIZE)
.spawn(|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.thread_stack_size(STACK_SIZE)
.enable_all()
.build()
.expect("MRF replay test runtime");
runtime.block_on(check_unknown_source_deleted_bucket_replay());
})
.expect("MRF replay test thread")
.join()
.expect("MRF replay test must finish");
}
async fn check_unknown_source_deleted_bucket_replay() {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("rustfs_mrf_unknown_deleted_bucket")
.build()
.await;
let bucket = "partial-write-retention";
env.make_bucket(bucket, false).await;
let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None));
let mut original = runtime();
for index in 0..3 {
original
.admit_partial_write(intent(&format!("deleted-bucket-{index}")), None)
.expect("admit each unknown-source responsibility");
}
assert!(original.flush().await, "commit responsibilities before deleting the bucket");
let before = original.partial_writes.checkpoint_records(super::super::super::intent_digest);
env.ecstore
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete the bucket through its supported API");
let mut restarted = runtime();
let replay = super::super::super::replay_into(&manager, &mut restarted.queue, &mut None).await;
assert_eq!(replay.partial_writes.len(), before.len());
restarted.next_checkpoint_sequence = replay.next_checkpoint_sequence;
restarted.replay_cleanup = replay.cleanup;
restarted.retain_replay_journal = replay.retain_journal_for_replay;
restarted.durable_replay_anchors = replay.durable_replay_anchors;
restarted.adopt_replayed_partial_writes(replay.partial_writes);
restarted.dirty = true;
assert!(restarted.flush().await, "publish the retained owner after replay");
restarted.dispatch(&manager).await;
let mut visible = Vec::new();
let mut cursor = None;
loop {
let page = restarted
.list_unverified_legacy(cursor, 1)
.expect("list readable responsibilities");
assert_eq!(page.legacy_generation_unknown_count, before.len());
visible.extend(page.responsibilities.into_iter().map(|row| {
assert_eq!(row.source_bucket_incarnation_id, None);
assert_eq!(row.bucket_incarnation_id, None);
row.responsibility_id
}));
cursor = page.next_cursor;
if cursor.is_none() {
break;
}
}
assert_eq!(
visible.into_iter().collect::<HashSet<_>>(),
before.iter().map(|row| row.responsibility_id).collect()
);
env.make_bucket(bucket, false).await;
let observed = env
.ecstore
.bucket_incarnation_id(bucket)
.await
.expect("recreated incarnation");
for entry in restarted.partial_writes.entries.values_mut() {
entry.next_attempt = Instant::now();
}
for _ in 0..before.len() {
restarted.dispatch(&manager).await;
}
let rows = restarted
.list_unverified_legacy(None, 10)
.expect("list observed responsibilities")
.responsibilities;
assert_eq!(rows.len(), before.len(), "identity probing cannot discharge a responsibility");
for row in rows {
assert_eq!(
row.status, "legacy_generation_unknown",
"recreated buckets must be observed automatically"
);
assert_eq!(row.bucket_incarnation_id, Some(observed));
assert_eq!(
row.source_bucket_incarnation_id, None,
"current identity is never evidence of the old source"
);
assert!(
restarted
.recheck_legacy_generation(&manager, row.responsibility_id, observed)
.await
.is_err()
);
}
assert_eq!(manager.get_queue_length().await, 0, "identity probes must not enqueue object healing");
}
crates/heal/src/heal/mrf_queue/tests/empty_events_tests.rs
+3 -1
View File
@@ -52,6 +52,7 @@ fn runtime() -> MrfRuntime {
dirty: false,
journal_on_disk: false,
retain_replay_journal: false,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: None,
@@ -425,7 +426,8 @@ fn empty_events_real_consumer_stop_retains_unresolved_journal() {
let listing = tokio::time::timeout(Duration::from_secs(10), listed)
.await
.expect("consumer startup finishes")
.expect("listing receipt");
.expect("listing receipt")
.expect("readable lifecycle snapshot");
assert!(
listing
.responsibilities
crates/heal/src/heal/mrf_queue/tests/lifecycle_replay_tests.rs
+258
View File
@@ -0,0 +1,258 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
fn lifecycle_frame(payload: &Value) -> Vec<u8> {
let payload = serde_json::to_vec(payload).expect("encode lifecycle fixture");
let mut encoded = Vec::from(MRF_LIFECYCLE_MAGIC.as_slice());
encoded.push(MRF_LIFECYCLE_FORMAT_VERSION);
encoded.extend_from_slice(&u64::try_from(payload.len()).expect("fixture size").to_le_bytes());
encoded.extend_from_slice(&sha2::Sha256::digest(&payload));
encoded.extend_from_slice(&payload);
encoded
}
#[test]
fn unknown_source_deleted_bucket_lifecycle_rejects_unknown_state_version_and_fields() {
assert_eq!(
serde_json::to_string(&partial_write::ResponsibilityState::Active {}).expect("encode active state"),
r#"{"state":"active"}"#,
"strict field validation must preserve the legacy encoding"
);
let owner = Uuid::new_v4();
let intent_digest = [7_u8; 32];
let mut payload = json!({
"format_version": MRF_LIFECYCLE_FORMAT_VERSION,
"checkpoint_owner": owner,
"checkpoint_sequence": 1,
"records": [{
"intent_digest": intent_digest,
"responsibility_id": Uuid::new_v4(),
"source_bucket_incarnation_id": null,
"last_operator_acceptance": null,
"state": {"state": "active"}
}]
});
let valid = lifecycle_frame(&payload);
assert!(decode_mrf_lifecycle_checkpoint(&valid, 8192).is_ok());
let mut future_version = valid;
future_version[8] += 1;
assert!(matches!(
decode_mrf_lifecycle_checkpoint(&future_version, 8192),
Err(MrfReplayError::Snapshot(snapshot::SnapshotError::Unsupported))
));
payload["records"][0]["state"] = json!({"state": "source_and_current_generation_unknown", "since_ms": 7});
assert!(matches!(
decode_mrf_lifecycle_checkpoint(&lifecycle_frame(&payload), 8192),
Err(MrfReplayError::Lifecycle(_))
));
payload["records"][0]["state"] = json!({"state": "active", "hidden_source_incarnation": Uuid::new_v4()});
assert!(matches!(
decode_mrf_lifecycle_checkpoint(&lifecycle_frame(&payload), 8192),
Err(MrfReplayError::Lifecycle(_))
));
}
fn durable_files(roots: &[PathBuf]) -> Vec<Option<Vec<u8>>> {
roots
.iter()
.flat_map(|root| {
[
".heal-mrf-snapshot.0.bin",
".heal-mrf-snapshot.1.bin",
".heal-mrf-commit.0.bin",
".heal-mrf-commit.1.bin",
MRF_LIFECYCLE_PATHS[0],
MRF_LIFECYCLE_PATHS[1],
MRF_SCOPED_JOURNAL_PATH,
MRF_JOURNAL_PATH,
]
.map(|path| fs::read(root.join(RUSTFS_META_BUCKET).join(path)).ok())
})
.collect()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unknown_source_deleted_bucket_unsupported_lifecycle_freezes_all_writes() {
assert_unreadable_lifecycle_is_retained(|payload| {
payload["records"][0]["state"] = json!({"state": "future_held", "since_ms": 17});
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unknown_source_deleted_bucket_mismatched_lifecycle_freezes_all_writes() {
assert_unreadable_lifecycle_is_retained(|payload| {
let wrong_digest = [7_u8; 32];
payload["records"][0]["intent_digest"] = json!(wrong_digest);
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unknown_source_deleted_bucket_missing_lifecycle_record_freezes_all_writes() {
assert_unreadable_lifecycle_is_retained(|payload| payload["records"] = json!([])).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unknown_source_deleted_bucket_extra_lifecycle_record_freezes_all_writes() {
assert_unreadable_lifecycle_is_retained(|payload| {
let mut extra = payload["records"][0].clone();
extra["responsibility_id"] = json!(Uuid::new_v4());
extra["source_bucket_incarnation_id"] = json!(Uuid::new_v4());
payload["records"].as_array_mut().expect("fixture records").push(extra);
})
.await;
}
async fn assert_unreadable_lifecycle_is_retained(corrupt: fn(&mut Value)) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("rustfs_mrf_unknown_lifecycle")
.build()
.await;
let config = MrfConsumerConfig::default();
let owner = Uuid::new_v4();
let mut record = intent("retained-bucket", "retained-object", 0);
record.kind = MrfKind::PartialWrite;
let bytes = encoded_payload(&record);
let mut lifecycle = json!({
"format_version": MRF_LIFECYCLE_FORMAT_VERSION,
"checkpoint_owner": owner,
"checkpoint_sequence": 1,
"records": [{
"intent_digest": intent_digest(&record).expect("fixture digest"),
"responsibility_id": Uuid::new_v4(),
"source_bucket_incarnation_id": Uuid::new_v4(),
"last_operator_acceptance": null,
"state": {"state": "active"}
}]
});
corrupt(&mut lifecycle);
let invalid_lifecycle = lifecycle_frame(&lifecycle);
snapshot::publish_committed_snapshot_with_companion(
&journal_disks().await,
owner,
1,
&bytes,
config.journal_max_bytes,
Some((&MRF_LIFECYCLE_PATHS, &invalid_lifecycle, config.journal_max_bytes)),
)
.await
.expect("commit an unreadable companion with the retained intent");
let before = durable_files(&env.disk_paths);
assert!(before.iter().any(Option::is_some), "fixture must contain actual committed files");
assert!(
read_replay_source(config.journal_max_bytes, config.journal_max_bytes * 4)
.await
.is_err(),
"an unreadable companion must not be treated as an absent legacy companion"
);
let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None));
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
assert!(replay.retain_journal_for_replay);
assert!(replay.partial_writes.is_empty(), "do not synthesize source-less active responsibilities");
let mut runtime = MrfRuntime {
partial_writes: PartialWrites::default(),
queue,
config,
checkpoint_owner: Uuid::new_v4(),
next_checkpoint_sequence: replay.next_checkpoint_sequence,
new_since_flush: 0,
dirty: true,
journal_on_disk: replay.journal_on_disk,
retain_replay_journal: replay.retain_journal_for_replay,
replay_readable: replay.source_readable,
durable_replay_anchors: replay.durable_replay_anchors,
replay_cleanup: replay.cleanup,
runtime_checkpoint: None,
backoff_until: None,
};
assert!(!runtime.flush().await, "an unreadable owner must fence every writer");
assert!(matches!(
runtime.admit_partial_write(record.clone(), Some(Uuid::new_v4())),
Err(MrfDurableAdmissionError::Persistence)
));
assert!(!runtime.flush().await, "new admission must not overwrite the unreadable owner");
assert!(!runtime.delete_idle_recovery_anchors().await, "idle cleanup must also remain fenced");
assert!(matches!(
runtime.list_unverified_legacy(None, 10),
Err(MrfLifecycleControlError::CheckpointUnreadable)
));
assert_eq!(
durable_files(&env.disk_paths),
before,
"payloads, manifests, companions and mirrors must stay byte-identical"
);
let (incoming, receiver) = mpsc::channel(4);
let (durable, durable_receiver) = mpsc::channel(4);
let (lifecycle, lifecycle_receiver) = mpsc::channel(4);
let consumer = tokio::spawn(run_mrf_consumer(manager.clone(), receiver, durable_receiver, lifecycle_receiver));
let (response, received) = tokio::sync::oneshot::channel();
durable
.send(MrfDurableSubmission {
intent: record,
source_bucket_incarnation_id: Some(Uuid::new_v4()),
response,
})
.await
.expect("submit to the actual consumer");
assert!(matches!(
tokio::time::timeout(Duration::from_secs(5), received)
.await
.expect("admission deadline")
.expect("admission response"),
Err(MrfDurableAdmissionError::Persistence)
));
let (response, listed) = tokio::sync::oneshot::channel();
lifecycle
.send(MrfLifecycleCommand::List {
after: None,
limit: 10,
response,
})
.await
.expect("query the actual consumer");
assert!(matches!(
tokio::time::timeout(Duration::from_secs(5), listed)
.await
.expect("listing deadline")
.expect("listing response"),
Err(MrfLifecycleControlError::CheckpointUnreadable)
));
incoming
.send(intent("retained-bucket", "new-best-effort", 0))
.await
.expect("queue a periodic flush");
tokio::time::sleep(Duration::from_millis(600)).await;
drop(incoming);
tokio::time::timeout(Duration::from_secs(5), consumer)
.await
.expect("consumer shutdown deadline")
.expect("consumer shutdown");
assert_eq!(manager.get_queue_length().await, 0, "unreadable recovery must not dispatch object work");
assert_eq!(
durable_files(&env.disk_paths),
before,
"startup, admission, periodic flush and shutdown must preserve every file"
);
}
crates/heal/src/heal/storage.rs
+7 -5
View File
@@ -1289,11 +1289,13 @@ impl HealStorageAPI for ECStoreHealStorage {
}
async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
self.ecstore
.bucket_incarnation_id(bucket)
.await
.map(Some)
.map_err(Error::Storage)
match self.ecstore.bucket_incarnation_id(bucket).await {
Ok(incarnation) => Ok(Some(incarnation)),
// A missing bucket leaves the current identity unresolved. It is
// not a retirement receipt for an unknown source generation.
Err(StorageError::BucketNotFound(_)) => Ok(None),
Err(error) => Err(Error::Storage(error)),
}
}
async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
crates/heal/tests/mrf_partial_write_test.rs
+1 -1
View File
@@ -173,7 +173,7 @@ async fn legacy_unbound_generation_is_parked_and_requires_explicit_risk_acceptan
assert!(
matches!(
mrf_queue::refresh_legacy_responsibility(entry.responsibility_id, None).await,
Err(rustfs_heal::heal::mrf_queue::MrfLifecycleControlError::IncarnationChanged)
Err(rustfs_heal::heal::mrf_queue::MrfLifecycleControlError::BucketIdentityUnresolved)
),
"an absent bucket is not retirement proof"
);
crates/rio/src/compress_index.rs
+47 -10
View File
@@ -220,7 +220,7 @@ impl Index {
b.extend_from_slice(S2_INDEX_HEADER);
// Add total sizes
let mut tmp = [0u8; 8];
let mut tmp = [0u8; 10];
let n = write_varint(&mut tmp, uncomp_total);
b.extend_from_slice(&tmp[..n]);
let n = write_varint(&mut tmp, comp_total);
@@ -472,31 +472,40 @@ impl Index {
}
}
// Helper functions for varint encoding/decoding
fn write_varint(buf: &mut [u8], mut v: i64) -> usize {
// Helper functions for signed varint encoding/decoding (ZigZag + LEB128,
// matching Go binary.PutVarint / binary.Varint used by the S2 index format).
fn write_varint(buf: &mut [u8], v: i64) -> usize {
let mut uv = ((v as u64) << 1) ^ ((v >> 63) as u64);
let mut n = 0;
while v >= 0x80 {
buf[n] = (v as u8) | 0x80;
v >>= 7;
while uv >= 0x80 {
buf[n] = (uv as u8) | 0x80;
uv >>= 7;
n += 1;
}
buf[n] = v as u8;
buf[n] = uv as u8;
n + 1
}
fn read_varint(buf: &[u8]) -> io::Result<(i64, usize)> {
let mut result = 0i64;
let mut shift = 0;
let mut uv = 0u64;
let mut shift = 0u32;
let mut n = 0;
while n < buf.len() {
let byte = buf[n];
n += 1;
result |= ((byte & 0x7F) as i64) << shift;
if n == 10 && byte > 1 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "varint overflow"));
}
uv |= ((byte & 0x7F) as u64) << shift;
if byte < 0x80 {
let result = ((uv >> 1) as i64) ^ (-((uv & 1) as i64));
return Ok((result, n));
}
shift += 7;
if shift >= 64 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "varint overflow"));
}
}
Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF"))
@@ -717,4 +726,32 @@ mod tests {
Ok(())
}
#[test]
fn test_index_round_trip_with_negative_compressed_deltas_and_large_offsets() -> io::Result<()> {
let mut source = Index::new();
source.est_block_uncomp = 1 << 20;
// Compressed blocks much smaller than est_block_uncomp / 2 (512 KiB)
// produce negative c_off residuals (-524,088 and -262,144).
source.add(0, 0)?;
source.add(200, 1 << 20)?;
source.add(300, 2 << 20)?;
source.add(450, (3 << 20) + 12345)?;
source.total_uncompressed = 1i64 << 56;
source.total_compressed = 1i64 << 55;
let encoded = source.clone().into_vec();
let mut decoded = Index::new();
let rest = decoded.load(encoded.as_ref())?;
assert!(rest.is_empty());
assert_eq!(decoded.total_uncompressed, source.total_uncompressed);
assert_eq!(decoded.total_compressed, source.total_compressed);
assert_eq!(decoded.info.len(), source.info.len());
for (actual, expected) in decoded.info.iter().zip(source.info.iter()) {
assert_eq!(actual.compressed_offset, expected.compressed_offset);
assert_eq!(actual.uncompressed_offset, expected.uncompressed_offset);
}
Ok(())
}
}
crates/scanner/src/scanner/cycle_state.rs
+1
View File
@@ -939,6 +939,7 @@ pub(super) async fn scanner_usage_recovery_intents_for_startup(
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(entry.name);
}
Ok(())
})
})),
..Default::default()
crates/scanner/src/scanner_folder.rs
+1
View File
@@ -2707,6 +2707,7 @@ impl FolderScanner {
"Scanner list_path_raw agreed callback failed"
);
}
Ok(())
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
docs/README.md
+2
View File
@@ -8,6 +8,8 @@ collection:
## Operations
- [Retained MRF responsibilities](operations/mrf-responsibilities.md) — unknown bucket identities, bounded probes, and unreadable checkpoint recovery.
- [Internode HTTP streaming transport](operations/internode-http-transport.md) — protocol selection, pooled connection isolation, and distributed read validation.
- [Persisted config compatibility](operations/persisted-config-compatibility.md) — ignored MinIO settings, bounded diagnostics, alias retention, and safe resets.
docs/operations/delete-transaction-recovery.md
+67 -2
View File
@@ -72,7 +72,9 @@ completed or canceled probes do not consume shared discovery cursors.
## Entry points
Every startup, including a clean restart, starts incremental raw discovery of
local transaction records. During normal operation, deletion registers a
local transaction records once the instance's bucket metadata system has been
published. Bootstrap is not classified as a failed disk scope and cannot
consume the initial discovery obligation. During normal operation, deletion registers a
bounded, coalesced exact-key retry hint before preparing metadata. Synchronous
completion clears only its own hint generation; an uncertain publication or
unfinished cleanup remains eligible for automatic replay without another
@@ -216,7 +218,21 @@ curl with `--image`, and a unique Docker namespace. The optional
`--baseline-binary` enables the mixed-version case. The harness uses synthetic
S3 objects, localhost ports, labeled isolated volumes, and a non-root server.
It stops only its own containers and keeps JSON results and logs in the chosen
artifact directory. Production builds must not enable `e2e-test-hooks`.
artifact directory. Production builds must not enable `e2e-test-hooks`. The `release-batch-crash`
case accepts a normal release binary: it observes persisted preparation before
SIGKILL in one round and persisted commit receipts in another, verifies automatic
mixed inline/external recovery without a new DELETE
or administrator request, and checks that a second clean restart is stable.
This case does not pin an individual instruction or transaction phase.
Physical settlement permits directory-only remnants that ordinary non-force
bucket removal safely prunes. It rejects retained payloads, backup or transaction
records, unknown files, and symbolic links. Final cleanup deletes only surviving
objects before requiring ordinary bucket removal. An interrupted proof GC can
also leave an empty transaction directory beside another drive's receipt.
Recovery retires only that exact empty leaf before republishing a replay
witness; a nonempty directory, including an empty nested prefix, remains
protected. No recursive removal grants transaction authority.
Run the preactivation/prepare/apply/decision process-stop cases, interrupted restore,
returning and untouched peers, both generation orderings, version/retention
@@ -224,3 +240,52 @@ and prefix protection, administrator dry-run/removal, response-loss,
permission/corruption, and mixed-version cases. Preserve the source identity
of both binaries with the results. A passing process-stop matrix does not
cover hardware power cuts, Windows runtime behavior, or DELETE throughput.
## Performance acceptance
Linux forward batch preparation admits at most eight independent keys per
local disk request. Duplicate keys, ancestor/descendant keys, noncanonical
paths, legacy operations, and rollback keep their ordered execution. Other
platforms retain serial preparation. The existing per-key metadata lease,
transaction publication, namespace ownership, and strict synchronization
barriers still apply to every admitted operation.
Use `scripts/test/delete_recovery_perf.py` for resource-matched release
comparisons. Supply separate binaries and build receipts; each receipt binds
`binary_sha256`, `source_head`, and the build fields `profile`, `opt_level`,
`target`, `rustc`, `features`, `default_features`, `lto`, and `codegen_units`.
Receipts must describe matching release builds at optimization level 3 without
`e2e-test-hooks`. Preserve any source patch and the build command with them.
For example:
```sh
python3 scripts/test/delete_recovery_perf.py \
--baseline-binary ./baseline/rustfs --baseline-receipt ./baseline/build.json \
--candidate-binary ./candidate/rustfs --candidate-receipt ./candidate/build.json \
--image rust:1.99.1-trixie --artifact-dir ./delete-performance-results \
--cpuset-cpus 2-5 --memory 4g
```
The runner uses an A1/B1/B2/A2 sequence with at least three repetitions per
cohort. It checks original bytes and ETags, physical inline/external layouts,
empty listings, physical cleanup, and bucket removal. Each sample also cleanly
restarts its isolated server and waits for the bounded raw discovery page on
every local disk. Startup observations retain whether discovery exceeded
thirty seconds, including for historical baselines whose bootstrap race enters
the thirty-second retry interval; the observation deadline is ninety seconds.
A candidate exceeding the thirty-second budget fails acceptance even when the
longer observation eventually completes.
This separates server readiness and local discovery service time from the
timed DELETE. DELETE begins no earlier than thirty-five seconds after restart
in either variant, keeping process age and early maintenance phases comparable.
Normal scanner/heal loops are disabled for this
controlled attribution; automatic transaction recovery remains enabled.
Cohorts share the same image, CPU set, memory limit, strict durability policy,
and workload. A baseline or candidate drift above ten percent makes the result
inconclusive and withholds improvement percentages. The artifact retains raw
samples and per-operation cumulative disk service times. These concurrent
service times are not an additive wall-clock decomposition. Four logical
volumes on one Docker host filesystem do not represent four independent
physical devices or establish hardware power-loss durability.
docs/operations/internode-http-transport.md
+1 -1
View File
@@ -22,7 +22,7 @@ HTTP/1.1 uses more concurrent connections than multiplexed HTTP/2. Size file-des
For validation, compare complete 100 MiB and 1 GiB GETs through every coordinator, verify lengths and hashes, and include concurrent GET, scanner, and heal activity. Record latency, stall errors, CPU, connection counts, and file descriptors. Include small objects and uploads when assessing a deployment's performance. Local transport tests exercise TLS ALPN, shared HTTP/2 backpressure, full-body delivery, and HTTP/1.1 idle reuse; they do not replace a distributed cluster run.
The distributed E2E case below starts four processes with four drives each and a generated CA. Both S3 and volume endpoints use verified TLS. It streams 32 KiB, 100 MiB and 1 GiB uploads from files, checks complete and unaligned Range responses through every coordinator, and repeats full reads with all four coordinators active concurrently. Each GET checks Content-Length, Content-Range where applicable, the received byte count and SHA-256. SDK GET retries are disabled, and captured node logs must contain no internode body stalls.
The distributed E2E case below starts four processes with four drives each and a generated CA. Both S3 and volume endpoints use verified TLS. It streams 32 KiB, 100 MiB and 1 GiB uploads from files, checks complete and unaligned Range responses through every coordinator, and repeats full reads with all four coordinators active concurrently. Each concurrent GET has its own supervised Tokio task and cooperative budget, so body hashing in one reader cannot consume the other readers' polling budget. Each GET retains its 180-second deadline and checks Content-Length, Content-Range where applicable, the received byte count and SHA-256. SDK GET retries are disabled, and captured node logs must contain no internode body stalls. Timeout errors identify PUT or GET, object size, coordinator/range where applicable, response stage and received bytes; completed operations are recorded as they finish. In CI, node logs remain beneath `RUSTFS_E2E_LOG_DIR` after fixture teardown.
```bash
export RUSTUP_TOOLCHAIN=stable
docs/operations/mrf-responsibilities.md
+33
View File
@@ -0,0 +1,33 @@
# Retained MRF responsibilities
**Use this when:** a legacy partial-write responsibility has no source bucket identity, or the MRF controller reports an unreadable checkpoint.
**Source of truth:** `crates/heal/src/heal/mrf_queue.rs` (`read_replay_source`, `MrfRuntime::flush`), `crates/heal/src/heal/mrf_queue/partial_write.rs` (`probe_unknown_generation`, `list_unverified_legacy`), and `rustfs/src/admin/handlers/heal.rs` (`MrfLegacyResponsibilitiesHandler`).
## Unknown bucket identity
`GET /rustfs/admin/v4/heal/mrf/responsibilities` lists retained responsibilities on the receiving node. Follow `nextCursor` until it is absent; the maximum page size is 256. Compare responsibility IDs and original pool/set scopes, rather than matching only bucket and object names.
`legacy_generation_unobserved` means the original source identity is unknown and no current identity has been observed. Both incarnation fields remain null. A missing bucket is not evidence that this responsibility can be acknowledged or removed.
These entries stay outside object-heal dispatch. After an initial 60-second delay, the consumer rotates through a bounded probe queue, attempts at most one identity lookup per dispatch round, and limits each lookup to one second. Failed or unresolved attempts use the existing bounded backoff, with a five-second minimum and a 60-second ceiling. The queue examination budget is `RUSTFS_HEAL_MRF_REPLAY_BATCH`.
The listing exposes `lastIdentityProbeResult` (`unresolved`, `unavailable`, or `known`), `lastIdentityProbeAtMs`, and `nextIdentityProbeAtMs` when available. These are process-local observations. Restart resets the probe delay; these timestamps do not establish when a responsibility was first retained.
When a same-name bucket becomes visible, its identity is recorded as the observed current incarnation in `legacy_generation_unknown`. The source stays null. This transition does not enqueue object repair or create a repair proof. Targeted recheck remains rejected because the original generation cannot be established.
An explicit refresh distinguishes a successful lookup with no identity from a lookup failure. The former returns an unresolved-identity error; permission, quorum, transport, and invalid-identity failures return service unavailable without exposing internal storage details. Neither case is reported as a proven incarnation change.
## Unreadable checkpoint
A committed lifecycle companion with an unknown state, unsupported version, invalid contents, mismatched owner/sequence, conflicting replicas, or no readable replica must not be treated as an absent legacy companion. A genuinely absent companion remains compatible with older snapshots.
If replay cannot read the committed owner, the consumer retains the original payloads, manifests, lifecycle companions, and compatibility mirrors. It rejects durable admission, lifecycle queries/actions, snapshot publication, and cleanup. It also stops object dispatch. The read failure is sticky for that process: restore a compatible reader or repair the storage fault, then restart and inspect the retained files again.
Do not delete a companion, journal, or commit manifest to clear this condition. An empty API response would be misleading; the API returns service unavailable while the owner is unreadable.
## Format and rollout boundary
The writer continues to emit lifecycle format version 1. A source-less `Active` record still means a parked legacy responsibility, and observing a current incarnation uses the existing `LegacyGenerationUnknown` state. No new persistent state is activated by identity probing.
A durable state for unknown source and current generations, including the first-retained time and probe schedule across restarts, needs a separate reader-first rollout. The existing heal-control and cross-pool protocol capabilities do not prove support for a new lifecycle state. New writers require evidence for every reader and the minimum supported rollback reader; do not downgrade past that reader or silently convert an unknown state to `Active`.
docs/operations/scanner-benchmark-runbook.md
+2
View File
@@ -558,6 +558,8 @@ scripts/run_scanner_validation_harness.sh \
For ad hoc per-node snapshots outside the harness window, use the by-host `awscurl` loop in [Reading Distributed Metrics](scanner-runtime-controls.md#reading-distributed-metrics); the metrics endpoint reports only the node that handles the request.
Host collectors are optional. Check `host-telemetry-status.tsv` before using their output: each started collector records its exit status, and a failure emits a warning while scanner status collection remains available. A missing collector row is not a zero measurement. Each run resets its host telemetry artifacts, including when collection is skipped or the interval is zero. If a scanner status request fails, the harness stops and waits for its started collectors while preserving the request failure exit code; collector statuses are recorded when the output file remains writable. On macOS the harness uses `iostat -d -w <interval> -c <samples>`; Linux retains `iostat -xz <interval> <samples>`. These are host/device aggregates, not RustFS process I/O or filesystem syscall counts. The initial iostat report may cover time since boot; do not treat it as an aligned workload interval. `fs_usage` still requires the appropriate macOS privileges. Without it, report filesystem-event counts as unavailable and keep CPU/progress observations separate from production-burst attribution.
### Bucket metrics freshness validation
Use the harness around a post-start bucket creation workload to cover the timing where scanner startup sees no buckets, a bucket is created afterwards, and the first metrics collection must not confuse a cold usage cache with real zero usage:
docs/testing/distributed-e2e.md
+1 -1
View File
@@ -112,7 +112,7 @@ The `distributed::multipart_fence_acceptance_test` cases separate remote storage
Lock-layer regressions also cover every four-node read-quorum ordering, an eight-node write quorum behind silent voters, and the initial fan-out bound for prompt replies. Distributed acquisition probes unstarted voters within its existing attempt budget so silent early peers cannot hide a reachable quorum. Vote thresholds, owner identities, heartbeat checks, and late-grant cleanup retain their existing contracts.
Faults match the test bucket or upload suffix in bounded control-packet bodies, preserving their signed bytes and trailers. Readiness requires an actual matching RPC, without waiting for voters that adaptive fan-out does not contact. Recovery cases release waiting packets while the request is active; quorum cases discard expired packets before restoring the peer. Replaying a packet after its caller has timed out is a separate late-delivery fault model.
Faults match the test bucket or upload suffix in bounded control-packet bodies, preserving their signed bytes and trailers. Abort injection decodes only the unary Delete routing fields and matches the exact multipart root, excluding child transaction cleanup. Readiness requires an actual matching RPC, without waiting for voters that adaptive fan-out does not contact. Recovery cases release waiting packets while the request is active; quorum cases discard expired packets before restoring the peer. Replaying a packet after its caller has timed out is a separate late-delivery fault model. The injected S3 operation has SDK retries disabled. After the fault is removed, ordinary cleanup must recover within 45 seconds, allowing the existing lease and drive-health recovery intervals; only classified ServiceUnavailable responses are polled, and the server's Retry-After advice is honored. The fault-free eight-node workload keeps single-attempt APIs throughout. Failed Complete uses the best-effort rollback contract in [erasure-coding.md](../architecture/erasure-coding.md#7-write-path-and-write-quorum). A below-quorum Abort can also leave partially deleted staging. Each matrix reuses one cluster and bucket, requiring fresh responses from every lock peer before the next row so circuit-breaker history cannot reduce its available votes. After a partial mutation, ordinary Abort must succeed or report an already retired session, ListParts must return NoSuchUpload, and every node must list no residual uploads. This verifies the Abort recovery merged in #8904 without weakening the delete quorum; it does not claim residue-free rollback of a failed Complete. Bucket-generation recreation is covered by the cancellation case.
The four-node EC:2 cases cover exactly quorum and quorum-minus-one for storage writes and exclusive locks (three votes) and ordinary Create shared locks (two votes), error-cause and retry headers, each multipart phase, and bucket recreation after client cancellation. A sixteen-shard EC12+4 case repeats all four metadata commit phases at exactly twelve and eleven available shards. The eight-node, eight-drive case checks the server-reported one-pool/four-set EC12+4 geometry, uploads 45 MiB objects using 8 MiB parts with two files and four parts in flight, verifies complete bytes through every node, and checks aborted and completed uploads leave no visible sessions.
rustfs/src/admin/handlers/heal.rs
+30 -3
View File
@@ -1734,6 +1734,10 @@ struct MrfLegacyResponsibilityActionResponse {
fn map_mrf_lifecycle_control_error(error: rustfs_heal::heal::mrf_queue::MrfLifecycleControlError) -> rustfs_s3_types::S3Error {
use rustfs_heal::heal::mrf_queue::MrfLifecycleControlError;
match error {
MrfLifecycleControlError::CheckpointUnreadable => admin_error(
S3ErrorCode::ServiceUnavailable,
"MRF checkpoint is unreadable; retain its files and restore a compatible reader",
),
MrfLifecycleControlError::Unavailable | MrfLifecycleControlError::Persistence => {
admin_error(S3ErrorCode::ServiceUnavailable, "MRF lifecycle operation could not be completed")
}
@@ -1747,6 +1751,14 @@ fn map_mrf_lifecycle_control_error(error: rustfs_heal::heal::mrf_queue::MrfLifec
S3ErrorCode::InvalidRequest,
"bucket incarnation changed; refresh the MRF responsibility listing",
),
MrfLifecycleControlError::BucketIdentityUnresolved => admin_error(
S3ErrorCode::InvalidRequest,
"current bucket identity is unresolved; the MRF responsibility remains held",
),
MrfLifecycleControlError::BucketIdentityUnavailable(_) => admin_error(
S3ErrorCode::ServiceUnavailable,
"current bucket identity is unavailable; retry after storage recovers",
),
MrfLifecycleControlError::InvalidAction(reason) => admin_error(S3ErrorCode::InvalidRequest, reason),
}
}
@@ -1961,9 +1973,9 @@ mod tests {
build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path,
encode_heal_start_success, encode_heal_task_status, execute_after_heal_start_preflight, heal_channel_response_items,
heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response,
map_heal_response, merge_peer_heal_statuses, parse_mrf_legacy_responsibility_query, peer_topology_complete,
query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission,
validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
map_heal_response, map_mrf_lifecycle_control_error, merge_peer_heal_statuses, parse_mrf_legacy_responsibility_query,
peer_topology_complete, query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status,
reject_heal_admission, validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
};
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
@@ -2086,6 +2098,21 @@ mod tests {
assert!(parse_mrf_legacy_responsibility_query(&uri).is_err(), "accepted invalid query {query}");
}
}
#[test]
fn mrf_lifecycle_errors_distinguish_unresolved_identity_and_storage_failure() {
use rustfs_heal::heal::mrf_queue::MrfLifecycleControlError;
let unresolved = map_mrf_lifecycle_control_error(MrfLifecycleControlError::BucketIdentityUnresolved);
assert_eq!(unresolved.code(), &S3ErrorCode::InvalidRequest);
let unavailable = map_mrf_lifecycle_control_error(MrfLifecycleControlError::BucketIdentityUnavailable(Box::new(
rustfs_heal::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "private storage detail")),
)));
assert_eq!(unavailable.code(), &S3ErrorCode::ServiceUnavailable);
assert!(!unavailable.to_string().contains("private storage detail"));
let unreadable = map_mrf_lifecycle_control_error(MrfLifecycleControlError::CheckpointUnreadable);
assert_eq!(unreadable.code(), &S3ErrorCode::ServiceUnavailable);
}
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::sync::mpsc;
use tokio::time::Duration;
rustfs/src/app/metadata_route.rs
+6 -2
View File
@@ -690,14 +690,18 @@ async fn call_list_object_versions(req: S3Request<Body>, bucket: String) -> S3Re
let request = req.map_input(|_| input);
let output = DefaultBucketUsecase::from_global()
.execute_list_object_versions_m(request)
.await?;
.await
.map_err(crate::error::normalize_read_quorum_error)?;
xml_response(&output.output)
}
async fn call_list_objects_v2(req: S3Request<Body>, bucket: String) -> S3Result<S3Response<Body>> {
let input = list_objects_v2_input(bucket, &req.uri, &req.headers)?;
let request = req.map_input(|_| input);
let output = DefaultBucketUsecase::from_global().execute_list_objects_v2m(request).await?;
let output = DefaultBucketUsecase::from_global()
.execute_list_objects_v2m(request)
.await
.map_err(crate::error::normalize_read_quorum_error)?;
xml_response(&output.output)
}
rustfs/src/app/object/put.rs
+3 -3
View File
@@ -18,7 +18,7 @@ use super::*;
use crate::app::trailer_adapter::trailer_source;
use crate::auth::{RUSTFS_MAX_CONTENT_LENGTH_QUERY, VerifiedPresignedRequest, parse_presigned_put_max_content_length};
use crate::error::{ClientBodyReadTimeout, UploadLimitExceeded, normalize_put_read_quorum_error};
use crate::error::{ClientBodyReadTimeout, UploadLimitExceeded, normalize_read_quorum_error};
static PUT_FAILURE_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(5_000);
#[cfg(test)]
@@ -1118,7 +1118,7 @@ impl PutObjectCompletion {
/// Record the failed completion and hand the error back to the caller.
fn fail_put_object(completion: PutObjectCompletion, err: S3Error) -> S3Error {
let err = if matches!(&completion, PutObjectCompletion::S3(_)) {
normalize_put_read_quorum_error(err)
normalize_read_quorum_error(err)
} else {
err
};
@@ -1211,7 +1211,7 @@ impl DefaultObjectUsecase {
pub async fn execute_put_object(&self, _fs: &FS, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
self.execute_put_object_boxed(_fs, req)
.await
.map_err(normalize_put_read_quorum_error)
.map_err(normalize_read_quorum_error)
}
fn execute_put_object_boxed<'a>(
rustfs/src/error.rs
+7 -8
View File
@@ -190,13 +190,12 @@ pub(crate) fn slow_down_read_api_error(err: StorageError) -> ApiError {
}
}
/// Normalize a read-quorum failure that escaped through a PutObject boundary.
/// Normalize a storage read-quorum failure at PUT and LIST boundaries.
///
/// Storage-level read quorum failures keep their `SlowDownRead` code for GET
/// compatibility. A write request that encounters the same failure must expose
/// the write-side retry contract instead, while retaining the typed source for
/// diagnostics and audit consumers.
pub(crate) fn normalize_put_read_quorum_error(mut err: S3Error) -> S3Error {
/// compatibility. PUT and LIST expose storage unavailability instead of
/// throttling, while retaining the typed source for diagnostics and audit consumers.
pub(crate) fn normalize_read_quorum_error(mut err: S3Error) -> S3Error {
let is_slow_down_read = matches!(
err.code(),
S3ErrorCode::Custom(code) if &**code == SLOW_DOWN_READ_CODE
@@ -2108,7 +2107,7 @@ mod tests {
headers.insert("retry-after", http::HeaderValue::from_static("2"));
err.set_headers(headers);
let err = normalize_put_read_quorum_error(normalize_put_read_quorum_error(err));
let err = normalize_read_quorum_error(normalize_read_quorum_error(err));
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(err.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(err.message(), Some("The service is unavailable. Please retry."));
@@ -2139,7 +2138,7 @@ mod tests {
let mut err = S3Error::with_message(code.clone(), "original message");
err.set_source(Box::new(StorageError::ErasureReadQuorum));
let original_status = err.status_code();
let err = normalize_put_read_quorum_error(err);
let err = normalize_read_quorum_error(err);
assert_eq!(err.code(), &code);
assert_eq!(err.message(), Some("original message"));
assert_eq!(err.status_code(), original_status);
@@ -2153,7 +2152,7 @@ mod tests {
if let Some(source) = source {
err.set_source(Box::new(source));
}
let err = normalize_put_read_quorum_error(err);
let err = normalize_read_quorum_error(err);
assert_eq!(err.code(), &S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()));
assert_eq!(err.message(), Some("original message"));
}
rustfs/src/storage/access.rs
+2 -2
View File
@@ -25,7 +25,7 @@ use crate::auth::{
parse_presigned_multipart_max_total_object_size, parse_presigned_put_max_content_length,
reject_unsigned_amz_headers_on_sigv4_request,
};
use crate::error::{ApiError, normalize_put_read_quorum_error};
use crate::error::{ApiError, normalize_read_quorum_error};
use crate::license::license_check;
use crate::shared_types::RemoteAddr;
use crate::storage::request_context::RequestContext;
@@ -3432,7 +3432,7 @@ impl S3Access for FS {
Ok(())
}
.await
.map_err(normalize_put_read_quorum_error)
.map_err(normalize_read_quorum_error)
}
/// Checks whether the PutObjectAcl request has accesses to the resources.
rustfs/src/storage/ecfs.rs
+12 -3
View File
@@ -1357,14 +1357,20 @@ impl S3 for FS {
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
record_s3_op(S3Operation::ListObjectVersions);
let usecase = s3_api::bucket_usecase_for(self);
usecase.execute_list_object_versions(req).await
usecase
.execute_list_object_versions(req)
.await
.map_err(crate::error::normalize_read_quorum_error)
}
#[instrument(level = "debug", skip(self, req))]
async fn list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
record_s3_op(S3Operation::ListObjects);
let usecase = s3_api::bucket_usecase_for(self);
usecase.execute_list_objects(req).await
usecase
.execute_list_objects(req)
.await
.map_err(crate::error::normalize_read_quorum_error)
}
#[instrument(level = "debug", skip(self, req))]
@@ -1372,7 +1378,10 @@ impl S3 for FS {
crate::hp_guard!("S3::list_objects_v2");
record_s3_op(S3Operation::ListObjectsV2);
let usecase = s3_api::bucket_usecase_for(self);
usecase.execute_list_objects_v2(req).await
usecase
.execute_list_objects_v2(req)
.await
.map_err(crate::error::normalize_read_quorum_error)
}
#[instrument(level = "debug", skip(self, req))]
rustfs/src/storage/ecfs_test.rs
+77
View File
@@ -67,6 +67,83 @@ mod tests {
}
}
#[test]
#[serial_test::serial]
fn list_apis_report_storage_read_quorum_as_unavailable() {
crate::app::gating_test_env::run_large_stack_test("list-quorum-surface", || async {
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use s3s::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
crate::app::gating_test_env::shared_gating_ambient().await;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let bucket = format!("list-quorum-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create listing bucket");
let set = &store.pools[0].disk_set[0];
let saved_disks = set.disks.read().await.clone();
{
let mut disks = set.disks.write().await;
for disk in disks.iter_mut().skip(1) {
*disk = None;
}
}
let fs = FS::new();
let errors = [
S3::list_objects(
&fs,
build_request(
ListObjectsInput {
bucket: bucket.clone(),
..Default::default()
},
Method::GET,
),
)
.await
.expect_err("V1 cannot list below storage read quorum"),
S3::list_objects_v2(
&fs,
build_request(
ListObjectsV2Input {
bucket: bucket.clone(),
..Default::default()
},
Method::GET,
),
)
.await
.expect_err("V2 cannot list below storage read quorum"),
S3::list_object_versions(
&fs,
build_request(
ListObjectVersionsInput {
bucket,
..Default::default()
},
Method::GET,
),
)
.await
.expect_err("versions cannot list below storage read quorum"),
];
*set.disks.write().await = saved_disks;
for error in errors {
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error.source().expect("retain storage diagnostics");
assert!(
matches!(
source.downcast_ref::<StorageError>(),
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
),
"the production LIST boundary must expose a typed read-quorum failure: {source}"
);
}
});
}
#[test]
fn test_fs_creation() {
let _fs = FS::new();
scripts/run_scanner_validation_harness.sh
+37 -6
View File
@@ -25,6 +25,7 @@ JQ_BIN="jq"
SKIP_HOST_TELEMETRY=false
RUSTFS_PID=""
TELEMETRY_PIDS=()
TELEMETRY_NAMES=()
usage() {
cat <<'USAGE'
@@ -167,6 +168,9 @@ setup_output() {
mkdir -p "$OUT_DIR/status"
mkdir -p "$OUT_DIR/heal"
mkdir -p "$OUT_DIR/metrics"
rm -f "$OUT_DIR/pidstat.txt" "$OUT_DIR/iostat.txt" "$OUT_DIR/mpstat.txt" \
"$OUT_DIR/host-telemetry-status.tsv"
printf 'collector\texit_status\n' >"$OUT_DIR/host-telemetry-status.tsv"
SUMMARY_CSV="$OUT_DIR/scanner-summary.csv"
echo "timestamp,primary_pressure,current_cycle_objects_scanned,current_cycle_directories_scanned,last_cycle_result,last_cycle_partial_reason,last_cycle_partial_source,lifecycle_transition_scanner_missed,source_work_missed_total,current_cycle_usage_saves,last_cycle_usage_saves,usage_dirty_pending_buckets,usage_last_cycle_dirty_buckets,usage_last_cycle_cleared_dirty_buckets,usage_last_save_result,usage_last_save_unix_secs,life_time_scan_cycle,life_time_scan_bucket_drive,life_time_scan_object,life_time_save_usage,heal_queue_length,heal_active_tasks,heal_scanner_queued,heal_admin_queued,heal_auto_heal_queued" >"$SUMMARY_CSV"
}
@@ -209,6 +213,7 @@ first_rustfs_pid() {
start_host_telemetry() {
TELEMETRY_PIDS=()
TELEMETRY_NAMES=()
if [[ "$SKIP_HOST_TELEMETRY" == "true" || "$INTERVAL_SECS" == "0" ]]; then
return
@@ -220,29 +225,54 @@ start_host_telemetry() {
if [[ -n "$pid" ]] && command -v pidstat >/dev/null 2>&1; then
pidstat -p "$pid" "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/pidstat.txt" 2>&1 &
TELEMETRY_PIDS+=("$!")
TELEMETRY_NAMES+=("pidstat")
fi
if command -v iostat >/dev/null 2>&1; then
iostat -xz "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/iostat.txt" 2>&1 &
if [[ "$(uname -s)" == "Darwin" ]]; then
iostat -d -w "$INTERVAL_SECS" -c "$SAMPLES" >"$OUT_DIR/iostat.txt" 2>&1 &
else
iostat -xz "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/iostat.txt" 2>&1 &
fi
TELEMETRY_PIDS+=("$!")
TELEMETRY_NAMES+=("iostat")
fi
if command -v mpstat >/dev/null 2>&1; then
mpstat "$INTERVAL_SECS" "$SAMPLES" >"$OUT_DIR/mpstat.txt" 2>&1 &
TELEMETRY_PIDS+=("$!")
TELEMETRY_NAMES+=("mpstat")
fi
}
wait_host_telemetry() {
local pid
finalize_host_telemetry() {
local run_status="${1:-0}" index pid status write_status=0
if [[ ${#TELEMETRY_PIDS[@]} -eq 0 ]]; then
return
fi
for pid in "${TELEMETRY_PIDS[@]}"; do
wait "$pid" || true
local collector_pids=("${TELEMETRY_PIDS[@]}")
local collector_names=("${TELEMETRY_NAMES[@]}")
TELEMETRY_PIDS=()
TELEMETRY_NAMES=()
if [[ "$run_status" != "0" ]]; then
for pid in "${collector_pids[@]}"; do
kill "$pid" 2>/dev/null || true
done
fi
for index in "${!collector_pids[@]}"; do
if wait "${collector_pids[$index]}"; then
status=0
else
status=$?
printf 'WARN: host telemetry collector %s exited with status %s\n' "${collector_names[$index]}" "$status" >&2 || true
fi
printf '%s\t%s\n' "${collector_names[$index]}" "$status" >>"$OUT_DIR/host-telemetry-status.tsv" || write_status=$?
done
return "$write_status"
}
scanner_status_url() {
@@ -501,9 +531,10 @@ main() {
write_metadata "$started_at"
capture_config_snapshots
trap 'harness_exit_status=$?; finalize_host_telemetry "$harness_exit_status" || true; exit "$harness_exit_status"' EXIT
start_host_telemetry
capture_status_series
wait_host_telemetry
finalize_host_telemetry
write_report
echo "Scanner validation artifacts written to $OUT_DIR"
scripts/test/delete_recovery_docker.py
+145 -6
View File
@@ -19,7 +19,7 @@ Logical disks share the Docker host filesystem; this tests process/RPC failure
and namespace safety, not independent hardware failure or power-loss durability.
"""
from pathlib import Path
import base64, hashlib, ipaddress, json, os, subprocess, tempfile, threading, time, uuid, xml.etree.ElementTree as ET
import base64, hashlib, inspect, ipaddress, json, os, subprocess, tempfile, threading, time, uuid, xml.etree.ElementTree as ET
from urllib.parse import quote
ROOT = None
PREFIX = None
@@ -36,6 +36,12 @@ def run(*args, check=True, **kwargs):
def docker(*args, check=True):
return run('docker', *args, check=check)
def has_retained_content(root):
"""Empty directory trees are pruned by non-force bucket removal."""
if root.is_symlink() or (root.exists() and not root.is_dir()):
return True
return any(path.is_symlink() or not path.is_dir() for path in root.rglob('*'))
class S3:
def __init__(self, port):
@@ -223,6 +229,12 @@ class Cluster:
def kill(self, index=0):
docker('kill', '--signal', 'KILL', self.names[index])
def stop_clean(self, index=0, timeout=60):
docker('stop', '--time', str(timeout), self.names[index])
state = json.loads(docker('inspect', '--format', '{{json .State}}', self.names[index]).stdout)
assert state['ExitCode'] == 0 and not state['OOMKilled'], state
return state
def restart(self, index=None):
for i in range(self.nodes) if index is None else [index]:
docker('start', self.names[i])
@@ -1035,9 +1047,7 @@ def automatic_clean_restart():
clear_barriers(cluster)
before = residue_inventory(cluster, bucket, key)
assert all(any(path.endswith('delete-quorum-committed') for path in drive['files']) for node in before for drive in node)
docker('stop', '--time', '60', cluster.names[0])
state = json.loads(docker('inspect', '--format', '{{json .State}}', cluster.names[0]).stdout)
assert state['ExitCode'] == 0 and not state['OOMKilled'], state
state = cluster.stop_clean()
repair = "from pathlib import Path; import json,os; roots=list(Path('/data').glob('d*/%s/%s')); dirs=[p for root in roots for p in root.iterdir() if p.is_dir() and (p/'part.1').is_file()]; assert len(dirs)==4; [os.chmod(p,0o755) for p in dirs]; print(json.dumps({'restored_payload_dirs':len(dirs)}))" % (bucket, key)
cluster.inspect_data(repair)
cluster.restart()
@@ -1248,6 +1258,135 @@ def admin_smoke():
finally:
cluster.remove()
def release_batch_crash():
"""Observe real journals before SIGKILL without requiring test-only hooks."""
import random
rounds = []
for iteration, phase in enumerate(('prepared', 'receipts')):
cluster = Cluster(f'release-batch-crash-{iteration}', drives=4)
try:
cluster.create()
client = cluster.clients[0]
bucket = 'release-batch-crash'
small = random.Random(2849).randbytes(4096)
large = random.Random(2850).randbytes(1024 * 1024)
payloads = {f'inline/object-{index:04d}': small for index in range(256)}
payloads.update({f'external/object-{index:04d}': large for index in range(32)})
confirmed = {'confirmed/inline': small, 'confirmed/external': large}
client.put_bucket(bucket)
etags = {}
for key, payload in {**payloads, **confirmed}.items():
client.put(bucket, key, payload)
etags[key] = client.last_headers['etag']
for key in confirmed:
client.delete(bucket, key)
wait_inventory(cluster, bucket, key, lambda inventory: all(not drive['exists'] for node in inventory for drive in node))
request = {}
def delete_batch():
try:
xml = ET.Element('Delete')
for key in payloads:
ET.SubElement(ET.SubElement(xml, 'Object'), 'Key').text = key
body = ET.tostring(xml)
request['response'] = client.request('POST', bucket + '?delete', body, {
'Content-Type': 'application/xml',
'Content-MD5': base64.b64encode(hashlib.md5(body).digest()).decode(),
}, expected=200)
except Exception as error:
request['exception'] = str(error)
counts = (
"from pathlib import Path\nimport json\n"
f"bucket={bucket!r}\n"
"print(json.dumps([{'prepared': len(list((d/bucket).rglob('delete-prepared'))), "
"'receipts': len(list((d/bucket).rglob('delete-quorum-committed')))} "
"for d in Path('/data').glob('d*')]))"
)
thread = threading.Thread(target=delete_batch, daemon=True)
thread.start()
deadline = time.monotonic() + 30
while True:
observation = cluster.inspect_data(counts)
if any(drive[phase] for node in observation for drive in node):
break
if not thread.is_alive() or time.monotonic() >= deadline:
raise AssertionError('the selected durable evidence must be observed in a live batch before the crash')
time.sleep(0.02)
assert thread.is_alive(), 'the observed batch must still be active before SIGKILL'
(ROOT / f'release-batch-crash-{iteration}-observation.json').write_text(json.dumps({
'observed_phase': phase, 'observed_before_kill': observation,
'request_active_before_kill': True,
}, indent=2))
cluster.kill(0)
thread.join(10)
assert not thread.is_alive(), 'the interrupted client must finish'
acknowledged = set(confirmed)
if 'response' in request:
_, body = request['response']
for item in ET.fromstring(body):
if item.tag.rsplit('}', 1)[-1] == 'Deleted':
acknowledged.update(child.text for child in item if child.tag.rsplit('}', 1)[-1] == 'Key')
physical_probe = (
"from pathlib import Path\nimport json\n" + inspect.getsource(has_retained_content) +
f"bucket={bucket!r};keys={list(payloads) + list(confirmed)!r}\n"
"record_names={'xl.meta.bkp','delete-prepared','delete-target','delete-quorum-committed','delete-commit-finished','xl.meta.delete-marker.rollback'}\n"
"result=[]\nfor d in Path('/data').glob('d*'):\n"
" root=d/bucket\n"
" result.append({'prepared':len(list(root.rglob('delete-prepared'))),"
"'receipts':len(list(root.rglob('delete-quorum-committed'))),"
"'records':sum(p.name in record_names or p.name.startswith(('reserve-delete-data.','recovery-delete-data.','delete-data.')) for p in root.rglob('*')),"
"'live':[key for key in keys if (root/key/'xl.meta').is_file()],"
"'residue':[key for key in keys if not (root/key/'xl.meta').is_file() and has_retained_content(root/key)]})\n"
"print(json.dumps(result))"
)
def wait_for_physical_settlement():
deadline = time.monotonic() + 90
while True:
drives = [drive for node in cluster.inspect_data(physical_probe) for drive in node]
if all(not drive['records'] and not drive['residue'] for drive in drives):
live = set(drives[0]['live'])
if all(set(drive['live']) == live for drive in drives):
return live
if time.monotonic() >= deadline:
raise TimeoutError('automatic recovery must converge physically before any new S3 DELETE, LIST, GET, or admin request')
time.sleep(0.05)
cluster.restart()
# Physical inspection precedes S3 listing, which can itself purge
# eligible residue and must not manufacture recovery evidence here.
present = wait_for_physical_settlement()
assert present <= payloads.keys()
assert not (present & acknowledged), 'acknowledged deletes cannot resurrect'
assert set(client.list(bucket)) == present
for key, payload in {**payloads, **confirmed}.items():
status, body = client.request('GET', bucket + '/' + key, expected=(200, 404))
assert (status == 200) == (key in present)
if status == 200:
assert body == payload, 'every surviving key must preserve complete original bytes'
assert client.last_headers.get('etag') == etags[key], 'rollback must preserve the original ETag'
cluster.stop_clean()
cluster.restart()
assert wait_for_physical_settlement() == present, 'a second recovery cannot change the settled key set'
assert set(client.list(bucket)) == present
for key in present:
assert client.get(bucket, key) == payloads[key], 'a second recovery must preserve complete bytes'
assert client.last_headers.get('etag') == etags[key], 'a second recovery must preserve the original ETag'
if present:
client.batch_delete(bucket, sorted(present))
assert client.list(bucket) == []
client.delete_bucket(bucket)
rounds.append({'observed_phase': phase, 'observed_before_kill': observation,
'acknowledged': len(acknowledged), 'present': len(present),
'absent': len(payloads) + len(confirmed) - len(present),
'complete_bytes_and_etags_verified': len(present), 'second_restart_stable': True})
(ROOT / 'release-batch-crash-progress.json').write_text(json.dumps(rounds, indent=2))
finally:
cluster.remove()
return {'case': 'release-batch-crash', 'status': 'passed', 'rounds': rounds,
'fault_model': 'SIGKILL after observing durable preparation or receipt; no deterministic instruction barrier'}
def large_batch_delete():
cluster = Cluster('batch-1000', nodes=4, drives=2)
try:
@@ -1358,7 +1497,7 @@ def main():
global ROOT, PREFIX, IMAGE, PORT, CANDIDATE, BASELINE
import argparse, re, sys
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--binary', required=True, type=Path, help='Linux candidate binary built with e2e-test-hooks')
parser.add_argument('--binary', required=True, type=Path, help='Linux candidate; deterministic barrier cases require e2e-test-hooks')
parser.add_argument('--baseline-binary', type=Path, help='Older Linux binary, required by mixed case')
parser.add_argument('--image', required=True, help='Local Linux image with Python 3 and the binary runtime libraries')
parser.add_argument('--artifact-dir', required=True, type=Path, help='New directory for logs and stage handshakes')
@@ -1385,7 +1524,7 @@ def main():
existing = docker(resource, 'ls', '-q', '--filter', 'label=rustfs.delete-recovery-test=' + PREFIX).stdout.strip()
if existing:
parser.error('namespace already owns Docker resources; choose a new namespace')
cases = {'automatic-clean-restart': automatic_clean_restart, 'unknown-fresh': unknown_predecessor_fresh_delete, 'batch1000': large_batch_delete, 'versions3000': large_version_history, 'admin-smoke': admin_smoke, 'legacy-alias': legacy_transaction_data_dir_alias, 'missing-offline': ordinary_missing_with_offline_peer, 'inverse': retained_decision_newer_undecided, 'deep': deep_key_crash, 'deep-large-stack': lambda : deep_key_crash(True), 'admin-prefix': admin_prefix_scoped, 'response-loss': receipt_response_loss, 'admin-object': lambda : admin_legacy_dry_run(True), 'smoke': smoke, 'legacy1': lambda : legacy_retry(1), 'legacy4': lambda : legacy_retry(4), 'restore': rollback_crash, 'retention': protected_version, 'version-applied': lambda : versioned_crash('APPLIED'), 'version-decision': lambda : versioned_crash('DECISION'), 'returning': returning_drive_new_generation, 'untouched': untouched_returning_peer, 'missing-version': missing_version_returning_peer, 'mixed': mixed_versions, 'admin': admin_legacy_dry_run, 'prefix': uuid_prefix_crash, 'corruption': corruption_repair, 'permission': permission_fault, 'error-prepared': lambda : injected_error('PREPARED'), 'error-applied': lambda : injected_error('APPLIED'), 'error-decision': lambda : injected_error('DECISION')}
cases = {'release-batch-crash': release_batch_crash, 'automatic-clean-restart': automatic_clean_restart, 'unknown-fresh': unknown_predecessor_fresh_delete, 'batch1000': large_batch_delete, 'versions3000': large_version_history, 'admin-smoke': admin_smoke, 'legacy-alias': legacy_transaction_data_dir_alias, 'missing-offline': ordinary_missing_with_offline_peer, 'inverse': retained_decision_newer_undecided, 'deep': deep_key_crash, 'deep-large-stack': lambda : deep_key_crash(True), 'admin-prefix': admin_prefix_scoped, 'response-loss': receipt_response_loss, 'admin-object': lambda : admin_legacy_dry_run(True), 'smoke': smoke, 'legacy1': lambda : legacy_retry(1), 'legacy4': lambda : legacy_retry(4), 'restore': rollback_crash, 'retention': protected_version, 'version-applied': lambda : versioned_crash('APPLIED'), 'version-decision': lambda : versioned_crash('DECISION'), 'returning': returning_drive_new_generation, 'untouched': untouched_returning_peer, 'missing-version': missing_version_returning_peer, 'mixed': mixed_versions, 'admin': admin_legacy_dry_run, 'prefix': uuid_prefix_crash, 'corruption': corruption_repair, 'permission': permission_fault, 'error-prepared': lambda : injected_error('PREPARED'), 'error-applied': lambda : injected_error('APPLIED'), 'error-decision': lambda : injected_error('DECISION')}
cases.update({'multipart-sidecars': completed_multipart_sidecars, 'multipart-integrity': lambda: completed_multipart_sidecars(True), 'multipart-sidecars-applied': lambda: completed_multipart_sidecars(False, True), 'multipart-integrity-applied': lambda: completed_multipart_sidecars(True, True)})
for stage in ('PREPARED', 'APPLIED', 'DECISION'):
cases['marker-batch-' + stage.lower()] = lambda stage=stage: marker_batch_crash(stage)
scripts/test/delete_recovery_perf.py
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Compare strict DELETE stages using isolated, resource-matched ABBA cohorts.
Both binaries require a JSON receipt with binary_sha256, source_head, and an
identical build object containing profile=release, opt_level=3, target, rustc,
features, default_features, lto, and codegen_units. Disk operation service times
include concurrent work; they are not an additive wall-clock decomposition.
"""
import argparse
import hashlib
import inspect
import json
from pathlib import Path
import random
import re
import statistics
import sys
import time
import uuid
import delete_recovery_docker as harness
def binary_digest(binary):
digest = hashlib.sha256()
with binary.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def binary_receipt(binary, receipt):
value = json.loads(receipt.read_text())
if value.get("binary_sha256") != binary_digest(binary):
raise ValueError(f"binary does not match its receipt: {binary}")
if not re.fullmatch(r"[0-9a-f]{40}", value.get("source_head", "")):
raise ValueError("receipt requires an exact source_head")
build = value["build"]
required = {"profile", "opt_level", "target", "rustc", "features", "default_features", "lto", "codegen_units"}
if not required <= build.keys() or build["profile"] != "release" or build["opt_level"] != 3:
raise ValueError("receipts must describe optimized release builds")
if not isinstance(build["features"], list) or "e2e-test-hooks" in build["features"]:
raise ValueError("performance receipts must exclude test-only hooks")
return value
def operation_snapshot(client):
_, body = client.request("GET", "rustfs/admin/v3/storageinfo", expected=200)
found = []
def visit(value):
if isinstance(value, dict):
if "api_calls" in value and "last_minute" in value:
found.append(value)
else:
for item in value.values():
visit(item)
elif isinstance(value, list):
for item in value:
visit(item)
visit(json.loads(body))
if len(found) != 4:
raise AssertionError("the stage probe requires four complete local disk metrics")
result = {}
for metric in found:
for operation, sample in metric["last_minute"].items():
item = result.setdefault(operation, {"count": 0, "ns": 0})
item["count"] += sample["count"]
item["ns"] += sample["acc_time_ns"]
return result, [metric["last_minute"].get("discover_delete_transactions", {}).get("count", 0) for metric in found]
def stage_delta(before, after):
result = {}
for operation in before.keys() | after.keys():
old = before.get(operation, {"count": 0, "ns": 0})
new = after.get(operation, {"count": 0, "ns": 0})
count, elapsed = new["count"] - old["count"], new["ns"] - old["ns"]
if count < 0 or elapsed < 0:
raise ValueError("the rolling metric window expired; stage attribution is invalid")
if count:
result[operation] = {"calls": count, "service_ms": elapsed / 1_000_000}
return result
def drift_percent(first, second):
if min(first, second) <= 0:
raise ValueError("benchmark durations must be positive")
return abs(first - second) / min(first, second) * 100
def measure(cluster, case, count, size, batch, repeat):
client = cluster.clients[0]
bucket = f"delete-perf-{case}-{repeat}"
payload = random.Random(2849 + size).randbytes(size)
keys = [f"object-{index:04d}" for index in range(count)]
client.put_bucket(bucket)
for key in keys:
client.put(bucket, key, payload)
etag = client.last_headers.get("etag")
assert etag, "PUT must acknowledge the object's ETag"
assert client.get(bucket, key) == payload, "complete bytes must match before deletion"
assert client.last_headers.get("etag") == etag
assert sorted(client.list(bucket)) == keys
inventory = harness.residue_inventory(cluster, bucket, keys[0])
for node in inventory:
for drive in node:
external = any(path.rsplit("/", 1)[-1] == "part.1" for path in drive["files"])
assert external == (size > 128 * 1024), "the fixture must exercise its declared physical layout"
clean_stop = cluster.stop_clean(timeout=30)
restart_started = time.perf_counter()
cluster.restart()
ready_seconds = time.perf_counter() - restart_started
deadline = time.monotonic() + 90
while True:
startup_operations, completed = operation_snapshot(client)
if all(count >= 1 for count in completed):
break
if time.monotonic() >= deadline:
raise TimeoutError("clean restart did not complete raw discovery on all four disks")
time.sleep(0.02)
# Each fixture has one bucket and fewer than 4096 raw entries per disk,
# so its complete initial sweep uses one bounded page on each disk.
assert completed == [1] * 4, "unexpected discovery scopes invalidate startup attribution"
startup = {
"clean_exit_code": clean_stop["ExitCode"],
"ready_seconds": ready_seconds,
"initial_discovery_complete_seconds": time.perf_counter() - restart_started,
"discovery_within_30_seconds": time.perf_counter() - restart_started <= 30,
"raw_discovery_operations": startup_operations["discover_delete_transactions"],
}
assert sorted(client.list(bucket)) == keys, "the clean restart must preserve the fixture"
# Compare DELETE at the same process age. A faster startup must not move
# only the candidate into earlier periodic maintenance phases.
time.sleep(max(0, 35 - (time.perf_counter() - restart_started)))
before, _ = operation_snapshot(client)
latencies = []
started = time.perf_counter()
if batch:
client.batch_delete(bucket, keys)
else:
for key in keys:
tick = time.perf_counter()
client.delete(bucket, key)
latencies.append(time.perf_counter() - tick)
elapsed = time.perf_counter() - started
after, _ = operation_snapshot(client)
if time.perf_counter() - restart_started >= 59:
raise ValueError("the process-age bound no longer protects the rolling operation window")
assert client.list(bucket) == []
leftovers = cluster.inspect_data(
"from pathlib import Path\nimport json\n" + inspect.getsource(harness.has_retained_content) +
f"keys={keys!r};bucket={bucket!r}\n"
"print(json.dumps([str(d / bucket / key) for d in Path('/data').glob('d*') "
"for key in keys if has_retained_content(d / bucket / key)]))"
)
assert all(not node for node in leftovers), "requested keys must retain no payload, metadata, proof, or non-directory entries"
client.delete_bucket(bucket)
return {
"case": case, "repeat": repeat, "objects": count, "object_bytes": size,
"wall_seconds": elapsed, "operations": stage_delta(before, after), "clean_restart": startup,
"single_median_ms": statistics.median(latencies) * 1000 if latencies else None,
"complete_bytes_etag_and_physical_cleanup": True,
}
def comparison_report(samples, drift_limit):
medians = {cohort: {case: statistics.median(item["wall_seconds"] for item in values if item["case"] == case)
for case in ("inline-batch", "inline-single", "external-batch")}
for cohort, values in samples.items()}
drifts = {case: {"baseline_percent": drift_percent(medians["A1"][case], medians["A2"][case]),
"candidate_percent": drift_percent(medians["B1"][case], medians["B2"][case])}
for case in medians["A1"]}
startup_ok = all(sample["clean_restart"]["discovery_within_30_seconds"]
for cohort in ("B1", "B2") for sample in samples[cohort])
valid = startup_ok and all(value <= drift_limit for pair in drifts.values() for value in pair.values())
report = {"verdict": "comparable" if valid else "inconclusive", "medians_seconds": medians,
"drift": drifts, "candidate_discovery_within_30_seconds": startup_ok}
if valid:
report["wall_time_reduction_percent"] = {
case: (1 - statistics.median([medians["B1"][case], medians["B2"][case]]) /
statistics.median([medians["A1"][case], medians["A2"][case]])) * 100
for case in medians["A1"]}
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
for variant in ("baseline", "candidate"):
parser.add_argument(f"--{variant}-binary", type=Path, required=True)
parser.add_argument(f"--{variant}-receipt", type=Path, required=True)
parser.add_argument("--image", required=True, help="Existing local image with Python 3 and curl")
parser.add_argument("--artifact-dir", type=Path, required=True)
parser.add_argument("--cpuset-cpus", required=True)
parser.add_argument("--memory", default="4g")
parser.add_argument("--port", type=int, default=19640)
parser.add_argument("--repetitions", type=int, default=3)
parser.add_argument("--drift-limit-percent", type=float, default=10)
args = parser.parse_args()
if args.repetitions < 3 or not 0 < args.drift_limit_percent <= 10:
parser.error("at least three repetitions and a drift limit in (0, 10] are required")
if not 1024 <= args.port <= 65532 or not re.fullmatch(r"[0-9,-]+", args.cpuset_cpus):
parser.error("invalid port or CPU set")
baseline = binary_receipt(args.baseline_binary, args.baseline_receipt)
candidate = binary_receipt(args.candidate_binary, args.candidate_receipt)
if baseline["build"] != candidate["build"]:
parser.error("baseline and candidate build configurations differ")
args.artifact_dir.mkdir(parents=True, exist_ok=False)
inputs = {"baseline": baseline, "candidate": candidate, "configuration": vars(args).copy()}
(args.artifact_dir / "inputs.json").write_text(json.dumps(inputs, default=str, indent=2))
image = harness.docker("image", "inspect", "--format", "{{.Id}}", args.image).stdout.decode().strip()
original_docker = harness.docker
def resource_matched_docker(*command, check=True):
if command and command[0] == "run" and "-d" in command:
command = (command[0], "--cpuset-cpus", args.cpuset_cpus, "--memory", args.memory, *command[1:])
return original_docker(*command, check=check)
harness.docker = resource_matched_docker
samples = {}
for cohort, binary in (("A1", args.baseline_binary), ("B1", args.candidate_binary),
("B2", args.candidate_binary), ("A2", args.baseline_binary)):
root = args.artifact_dir / cohort
root.mkdir()
harness.ROOT, harness.PREFIX = root, f"delete-perf-{uuid.uuid4().hex[:12]}"
harness.IMAGE, harness.PORT = image, args.port
harness.CANDIDATE, harness.BASELINE = binary.resolve(), None
# Keep normal scanner/heal work out of stage attribution. Automatic
# transaction recovery is initialized by ECStore and remains enabled.
cluster = harness.Cluster("strict", drives=4, extra_env={
"RUSTFS_SCANNER_ENABLED": "false", "RUSTFS_HEAL_ENABLED": "false",
})
samples[cohort] = []
try:
cluster.create()
for repeat in range(args.repetitions):
for case, count, size, batch in (("inline-batch", 500, 4096, True),
("inline-single", 32, 4096, False),
("external-batch", 32, 1024 * 1024, True)):
result = measure(cluster, case, count, size, batch, repeat)
samples[cohort].append(result)
(root / "samples.json").write_text(json.dumps(samples[cohort], indent=2))
print(json.dumps({"cohort": cohort, **result}), flush=True)
finally:
cluster.remove()
binary_receipt(args.baseline_binary, args.baseline_receipt)
binary_receipt(args.candidate_binary, args.candidate_receipt)
report = comparison_report(samples, args.drift_limit_percent)
report["resource_contract"] = {
"cpuset_cpus": args.cpuset_cpus, "memory": args.memory, "image": image,
"topology": "1 process / 4 logical drives on one Docker host filesystem",
}
(args.artifact_dir / "result.json").write_text(json.dumps(report, indent=2))
print(json.dumps(report), flush=True)
return 0 if report["verdict"] == "comparable" else 2
if __name__ == "__main__":
sys.exit(main())
scripts/test/test_delete_recovery_perf.py
+93
View File
@@ -0,0 +1,93 @@
"""Regression checks for benchmark evidence and drift verdicts."""
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from types import SimpleNamespace
import delete_recovery_perf as perf
class EvidenceTests(unittest.TestCase):
def test_only_empty_directory_trees_are_not_payload_residue(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "key"
(root / "transaction").mkdir(parents=True)
self.assertFalse(perf.harness.has_retained_content(root))
proof = root / "transaction" / "xl.meta.bkp"
proof.write_bytes(b"backup")
self.assertTrue(perf.harness.has_retained_content(root))
proof.unlink()
payload = root / "unknown-entry"
payload.write_bytes(b"payload")
self.assertTrue(perf.harness.has_retained_content(root))
def test_symlinks_cannot_be_mistaken_for_empty_directories(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "key"
root.mkdir()
(root / "link").symlink_to(Path(directory) / "missing-target")
self.assertTrue(perf.harness.has_retained_content(root))
def test_clean_restart_rejects_forced_exit_or_oom(self):
for state in ({"ExitCode": 137, "OOMKilled": False}, {"ExitCode": 0, "OOMKilled": True}):
cluster = perf.harness.Cluster.__new__(perf.harness.Cluster)
cluster.names = ["owned-test-server"]
with patch.object(perf.harness, "docker", side_effect=[SimpleNamespace(), SimpleNamespace(stdout=json.dumps(state).encode())]):
with self.assertRaises(AssertionError):
cluster.stop_clean()
def test_expired_operation_window_cannot_be_attributed_to_the_candidate(self):
with self.assertRaisesRegex(ValueError, "expired"):
perf.stage_delta({"prepare": {"count": 4, "ns": 100}}, {"prepare": {"count": 3, "ns": 90}})
def test_operation_service_times_keep_their_units(self):
self.assertEqual(perf.stage_delta({}, {"prepare": {"count": 4, "ns": 2_000_000}}),
{"prepare": {"calls": 4, "service_ms": 2.0}})
def test_binary_identity_and_release_configuration_are_required(self):
with tempfile.TemporaryDirectory() as directory:
binary, receipt = Path(directory) / "binary", Path(directory) / "receipt.json"
binary.write_bytes(b"candidate")
value = {"binary_sha256": perf.binary_digest(binary), "source_head": "a" * 40,
"build": {"profile": "debug", "opt_level": 0}}
receipt.write_text(json.dumps(value))
with self.assertRaisesRegex(ValueError, "release"):
perf.binary_receipt(binary, receipt)
binary.write_bytes(b"different executable")
with self.assertRaisesRegex(ValueError, "does not match"):
perf.binary_receipt(binary, receipt)
def test_control_or_candidate_drift_withholds_every_improvement_claim(self):
for unstable in ("A2", "B2"):
samples = self.samples()
for sample in samples[unstable]:
sample["wall_seconds"] *= 2
result = perf.comparison_report(samples, 10)
self.assertEqual(result["verdict"], "inconclusive")
self.assertNotIn("wall_time_reduction_percent", result)
def test_candidate_startup_timeout_withholds_improvement_claims(self):
samples = self.samples()
samples["B1"][0]["clean_restart"]["discovery_within_30_seconds"] = False
result = perf.comparison_report(samples, 10)
self.assertFalse(result["candidate_discovery_within_30_seconds"])
self.assertEqual(result["verdict"], "inconclusive")
self.assertNotIn("wall_time_reduction_percent", result)
def test_stable_controls_allow_a_quantified_result(self):
result = perf.comparison_report(self.samples(), 10)
self.assertEqual(result["verdict"], "comparable")
self.assertAlmostEqual(result["wall_time_reduction_percent"]["inline-batch"], 30)
@staticmethod
def samples():
return {cohort: [{"case": case, "wall_seconds": value, "clean_restart": {"discovery_within_30_seconds": True}} for case in
("inline-batch", "inline-single", "external-batch") for _ in range(3)]
for cohort, value in (("A1", 10), ("B1", 7), ("B2", 7), ("A2", 10))}
if __name__ == "__main__":
unittest.main()
scripts/test_scanner_validation_harness.sh
+122 -2
View File
@@ -15,6 +15,9 @@ cat >"$BIN_DIR/mc" <<'STUB'
set -euo pipefail
printf '%s\n' "$*" >>"${MC_LOG:?}"
if [[ "${CONFIG_EXIT_CODE:-0}" != "0" ]]; then
exit "$CONFIG_EXIT_CODE"
fi
if [[ "$*" == "admin config get rustfs-local scanner" ]]; then
printf 'scanner delay="30" max_wait="15"\n'
elif [[ "$*" == "admin config get rustfs-local heal" ]]; then
@@ -29,6 +32,19 @@ cat >"$BIN_DIR/awscurl" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
if [[ "${SCANNER_STATUS_EXIT_CODE:-0}" != "0" && "$*" == */scanner/status* ]]; then
for attempt in {1..100}; do
[[ -s "${IOSTAT_PID_FILE:?}" ]] && break
sleep 0.05
done
test -s "$IOSTAT_PID_FILE"
if [[ -n "${SCANNER_STATUS_WRITE_FAIL_FILE:-}" ]]; then
rm -f "$SCANNER_STATUS_WRITE_FAIL_FILE"
mkdir "$SCANNER_STATUS_WRITE_FAIL_FILE"
fi
exit "$SCANNER_STATUS_EXIT_CODE"
fi
printf '%s\n' "$*" >>"${AWSCURL_LOG:?}"
printf 'access-env-present=%s\n' "${AWS_ACCESS_KEY_ID:+yes}" >>"${AWSCURL_LOG:?}"
printf 'secret-env-present=%s\n' "${AWS_SECRET_ACCESS_KEY:+yes}" >>"${AWSCURL_LOG:?}"
@@ -133,9 +149,19 @@ cat >"$BIN_DIR/pidof" <<'STUB'
exit 1
STUB
cat >"$BIN_DIR/uname" <<'STUB'
#!/usr/bin/env bash
printf '%s\n' "${TEST_UNAME:-Linux}"
STUB
cat >"$BIN_DIR/iostat" <<'STUB'
#!/usr/bin/env bash
printf 'iostat sample\n'
printf 'iostat args: %s\n' "$*"
if [[ -n "${IOSTAT_PID_FILE:-}" ]]; then
printf '%s\n' "$$" >"$IOSTAT_PID_FILE"
exec sleep 60
fi
exit "${IOSTAT_EXIT_CODE:-0}"
STUB
cat >"$BIN_DIR/mpstat" <<'STUB'
@@ -184,7 +210,7 @@ printf 'unexpected jq args: %s\n' "$*" >&2
exit 1
STUB
chmod +x "$BIN_DIR/mc" "$BIN_DIR/awscurl" "$BIN_DIR/pidof" "$BIN_DIR/iostat" "$BIN_DIR/mpstat" "$BIN_DIR/jq"
chmod +x "$BIN_DIR/uname" "$BIN_DIR/mc" "$BIN_DIR/awscurl" "$BIN_DIR/pidof" "$BIN_DIR/iostat" "$BIN_DIR/mpstat" "$BIN_DIR/jq"
mc_log="$TMP_DIR/mc.log"
awscurl_log="$TMP_DIR/awscurl.log"
@@ -271,6 +297,100 @@ test -s "$missing_pid_out/status/scanner-status.1."*.json
test -s "$missing_pid_out/iostat.txt"
test -s "$missing_pid_out/mpstat.txt"
grep -q 'iostat args: -xz 1 1' "$missing_pid_out/iostat.txt"
for platform in Darwin Linux; do
platform_out="$TMP_DIR/out-$platform"
TEST_UNAME="$platform" RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" \
PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local --endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin --samples 1 --interval-secs 1 --out-dir "$platform_out"
if [[ "$platform" == Darwin ]]; then
grep -q 'iostat args: -d -w 1 -c 1' "$platform_out/iostat.txt"
else
grep -q 'iostat args: -xz 1 1' "$platform_out/iostat.txt"
fi
test "$(grep -cx $'iostat\t0' "$platform_out/host-telemetry-status.tsv")" -eq 1
done
failed_config_out="$TMP_DIR/out-failed-config"
mkdir -p "$failed_config_out"
cp "$TMP_DIR/out-Linux/iostat.txt" "$TMP_DIR/out-Linux/mpstat.txt" \
"$TMP_DIR/out-Linux/host-telemetry-status.tsv" "$failed_config_out/"
printf 'previous pidstat sample\n' >"$failed_config_out/pidstat.txt"
config_status=0
CONFIG_EXIT_CODE=43 RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" \
PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local --endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin --samples 1 --interval-secs 1 --out-dir "$failed_config_out" \
>"$TMP_DIR/failed-config.log" 2>&1 || config_status=$?
test "$config_status" -eq 43
if [[ "$(cat "$failed_config_out/host-telemetry-status.tsv")" != $'collector\texit_status' ]]; then
echo "Expected output initialization to reset host telemetry before config capture" >&2
exit 1
fi
test ! -e "$failed_config_out/pidstat.txt"
test ! -e "$failed_config_out/iostat.txt"
test ! -e "$failed_config_out/mpstat.txt"
for status_sink in file directory; do
failed_request_out="$TMP_DIR/out-failed-request-$status_sink"
collector_pid_file="$TMP_DIR/collector-$status_sink.pid"
status_write_fail_file=""
if [[ "$status_sink" == directory ]]; then
status_write_fail_file="$failed_request_out/host-telemetry-status.tsv"
fi
request_status=0
SCANNER_STATUS_EXIT_CODE=42 IOSTAT_PID_FILE="$collector_pid_file" \
SCANNER_STATUS_WRITE_FAIL_FILE="$status_write_fail_file" \
RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" \
PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local --endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin --samples 1 --interval-secs 1 --out-dir "$failed_request_out" \
>"$TMP_DIR/failed-request-$status_sink.log" 2>&1 || request_status=$?
test "$request_status" -eq 42
collector_pid="$(cat "$collector_pid_file")"
if kill -0 "$collector_pid" 2>/dev/null; then
kill "$collector_pid" 2>/dev/null || true
echo "Expected a failed status request to stop and reap its host collector" >&2
exit 1
fi
if [[ "$status_sink" == file ]]; then
grep -q $'iostat\t143' "$failed_request_out/host-telemetry-status.tsv"
else
test -d "$failed_request_out/host-telemetry-status.tsv"
fi
grep -q 'host telemetry collector iostat exited with status 143' "$TMP_DIR/failed-request-$status_sink.log"
done
# Reuse both populated OS outputs with collection disabled in different ways.
for platform in Darwin Linux; do
platform_out="$TMP_DIR/out-$platform"
if [[ "$platform" == Darwin ]]; then
disabled_args=(--interval-secs 1 --skip-host-telemetry)
else
disabled_args=(--interval-secs 0)
fi
printf 'previous pidstat sample\n' >"$platform_out/pidstat.txt"
TEST_UNAME="$platform" RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" \
PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local --endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin --samples 1 "${disabled_args[@]}" --out-dir "$platform_out"
if [[ "$(cat "$platform_out/host-telemetry-status.tsv")" != $'collector\texit_status' ]]; then
echo "Expected disabled collection to reset the previous run's collector statuses" >&2
exit 1
fi
test ! -e "$platform_out/pidstat.txt"
test ! -e "$platform_out/iostat.txt"
test ! -e "$platform_out/mpstat.txt"
test -s "$platform_out/scanner-summary.csv"
done
failed_collector_out="$TMP_DIR/out-failed-collector"
IOSTAT_EXIT_CODE=7 RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" \
PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local --endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin --samples 1 --interval-secs 1 --out-dir "$failed_collector_out" \
2>"$TMP_DIR/collector-warning.txt"
grep -q $'iostat\t7' "$failed_collector_out/host-telemetry-status.tsv"
grep -q 'host telemetry collector iostat exited with status 7' "$TMP_DIR/collector-warning.txt"
test -s "$failed_collector_out/scanner-summary.csv"
missing_args_log="$TMP_DIR/missing-args.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local >"$missing_args_log" 2>&1; then
echo "Expected missing required arguments to fail" >&2