mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-10 12:19:07 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
1b2c1fcc1b |
@@ -193,7 +193,7 @@ jobs:
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest-xlarge","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
|
||||
|
||||
Generated
+40
-49
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "archive-trait"
|
||||
version = "0.0.15"
|
||||
version = "0.0.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "999bf88cc483664cab77700ca2fdfb0415a73412abe284ef5bc2319d00938e00"
|
||||
checksum = "0b5040ca935c09888e7dfb04383b4d62fa3571e25bc66b6f345b591e97958524"
|
||||
dependencies = [
|
||||
"cap-std",
|
||||
"thiserror 2.0.21",
|
||||
@@ -811,9 +811,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-nats"
|
||||
version = "0.51.0"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4f92759f418774b78c36ef07885849150ce950c67d9bcdeff8a9c253f57d9a1"
|
||||
checksum = "015f0a7a3b0b29701b78c76a4d4a122624c97c71fa30363381664eb729d619c7"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64 0.22.1",
|
||||
@@ -3906,9 +3906,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9"
|
||||
version = "0.5.7"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f3298a54fd82af243c9f122595762f5b0e48bd3d1ea8fe3f3eb5e95c130e881"
|
||||
checksum = "c83757d61dc75b3e9d3579a3004c7fe0c42aff2f779ca9eeec7f0c8e1ae3fc10"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"dial9-core",
|
||||
@@ -3990,9 +3990,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.5.7"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0428cba60fbcf37ab82666f3a7ef3b6d0671e90dfb6a352376813b471319f061"
|
||||
checksum = "6f0eb05627c12c57e40510e63f9a3d5f628ab4b32fec9ccdb0ae98544c13f15e"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"bytes",
|
||||
@@ -4525,9 +4525,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
version = "1.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
@@ -4594,7 +4594,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
|
||||
dependencies = [
|
||||
"io-lifetimes 2.0.4",
|
||||
"rustix",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5385,9 +5385,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.30.3"
|
||||
version = "0.31.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935"
|
||||
checksum = "4039758c8bca06a34c475d17aaafc17ac1073ce2cecd8dba8c82e69edb300bbe"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5420,15 +5420,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.30.3"
|
||||
version = "0.31.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "738324f75aec847dea984f6806402f4f1bdfab437b4700fc8c3ebcad2dff6424"
|
||||
checksum = "bbba0a3badfb9ebf95f22dc6cdb4afcd0ff07b91a85fa3196fd1060576ea3065"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.30.3"
|
||||
version = "0.31.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a"
|
||||
checksum = "7d980014875338771b00ac44f1c9e2ea1a10650aa68ce54226233bc767714361"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6412,9 +6412,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.24.0"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490750560fa7851fbbf5d3ab1a8750f09350ed361ed8aed85b1e1e40587812ee"
|
||||
checksum = "ef1d73c59a56771343712b22ea5615092c612c16ba291c0979e3105dfeec4890"
|
||||
dependencies = [
|
||||
"sha2 0.11.1",
|
||||
]
|
||||
@@ -10336,18 +10336,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.7.3"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "216ecfc9341b5d6e2506d47ce57c9232ae1b8020bb10264ea05286fb234286e7"
|
||||
checksum = "fc7914232a854336c62058c24a8362c553ac8e7adba2e2395ce9097db24d1510"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.7.3"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb16ad68f96883407a75cdb88252b43dc2ea592428f0fc54ff7885756de4b814"
|
||||
checksum = "8073624242781448512682f1931624ee67108b6b7873b22fb29c1a1df689ac0c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
@@ -11705,9 +11705,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.2"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
|
||||
checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -12037,9 +12037,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.4"
|
||||
version = "1.16.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
|
||||
checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -12234,9 +12234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "starshard"
|
||||
version = "2.4.0"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc45de5060fab0de00d7a42230877543ae00c37caf6f49975e69851089b023ac"
|
||||
checksum = "b0883700b50fd0b62120ebc4c07230e02f0cf1ead474c3b3b44eaa13e0a999ea"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hashbrown 0.17.1",
|
||||
@@ -12485,9 +12485,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-codec"
|
||||
version = "0.0.15"
|
||||
version = "0.0.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9dad615e50d181afea4bafea23dacb8e33ed52dde06503813570634b932d467"
|
||||
checksum = "6875ba507bbec6022500f46477159a03f07ebdffdb4ea043254974531df1dac2"
|
||||
dependencies = [
|
||||
"archive-trait",
|
||||
"tar-framing",
|
||||
@@ -12497,9 +12497,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-framing"
|
||||
version = "0.0.15"
|
||||
version = "0.0.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "563e8181b47c9cdcce569eae3ad431eec71543d6269dd60c7bd2350f70bcd233"
|
||||
checksum = "49727b1a300514960d425d0e741a7da3e693170f3db39de3180fdbbc07e542c5"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
"tokio",
|
||||
@@ -12873,9 +12873,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "1.2.0+spec-1.1.0"
|
||||
version = "1.2.1+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5ffcdf8f50436dbbfd675c1fe1b8e0854f42b7defea8584bdebe1aee900e561"
|
||||
checksum = "d5755c51863cdac711f1956baf8eb9588e060755f7d3c4c979e70ea1e1f32cd7"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"serde_core",
|
||||
@@ -12888,9 +12888,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053"
|
||||
checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -13783,15 +13783,6 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
@@ -13964,7 +13955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14214,9 +14205,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.3.0"
|
||||
version = "9.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426"
|
||||
checksum = "0700e6c427aff64acdbf02e85fbfd946c2ee12319d1af013092f3a2178914ff8"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"bzip2",
|
||||
|
||||
+4
-4
@@ -149,7 +149,7 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
@@ -249,8 +249,8 @@ anyhow = "1.0.104"
|
||||
arc-swap = "1.9.3"
|
||||
rustfs-tokio-tar = { version = "0.7.1" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.15"
|
||||
tar-framing = "0.0.15"
|
||||
tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
@@ -389,7 +389,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
hotpath = { version = "0.31.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -17700,16 +17700,6 @@ impl ECStore {
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn complete_scanner_pause_backlog_retirement_for_test(&self, source_pool_index: usize) -> Result<()> {
|
||||
// Native backlog fixtures use bounded capacity inputs rather than the
|
||||
// host filesystem's unrelated usage. Keep the real ILM completion gate.
|
||||
let run_token = self.durable_ilm_receipt_run_token(source_pool_index).await?;
|
||||
self.persist_decommission_durable_ilm_manifest_for_run(source_pool_index, &run_token)
|
||||
.await?;
|
||||
self.complete_decommission(source_pool_index).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn retire_scanner_pause_backlog_for_test(
|
||||
self: &Arc<Self>,
|
||||
|
||||
@@ -85,7 +85,6 @@ impl ScannerPauseBacklogRetirementPhase {
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
pub set_id: Option<uuid::Uuid>,
|
||||
/// False for retiring members retained only to verify the old commit proof.
|
||||
pub writable: bool,
|
||||
pub data: Option<Vec<u8>>,
|
||||
@@ -141,7 +140,6 @@ async fn read_replica((set, writable): (Arc<SetDisks>, bool)) -> Result<ScannerP
|
||||
let mut replica = ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
set_id: Some(set.persistent_set_id()?),
|
||||
writable,
|
||||
data: None,
|
||||
};
|
||||
@@ -224,7 +222,6 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
.map(|read| ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: read.replica.pool_index,
|
||||
set_index: read.replica.set_index,
|
||||
set_id: read.replica.set_id,
|
||||
writable: read.replica.writable,
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
|
||||
@@ -685,8 +685,12 @@ async fn clear_bucket_usage_memory(bucket: &str, guard: Option<&rustfs_lock::Nam
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
remove_bucket_usage_for_namespace_change(store.as_ref(), bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
|
||||
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
|
||||
remove_bucket_usage_with_owned_namespace(store, bucket, None).await
|
||||
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
||||
@@ -726,37 +730,11 @@ where
|
||||
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(), Error> {
|
||||
let owned_store = crate::bucket::metadata_sys::object_store_in(&store.ctx).await?;
|
||||
if !std::ptr::eq(store, owned_store.as_ref()) {
|
||||
return Err(Error::other("usage cleanup requires its own metadata store"));
|
||||
}
|
||||
remove_bucket_usage_with_owned_namespace(owned_store, bucket, guard.cloned()).await
|
||||
}
|
||||
|
||||
async fn remove_bucket_usage_with_owned_namespace(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> Result<(), Error> {
|
||||
let bucket = bucket.to_owned();
|
||||
let ctx = Arc::clone(&store.ctx);
|
||||
// Retain the bucket generation and movement/object guards through every
|
||||
// physical commit even when the namespace request stops waiting.
|
||||
ctx.run_detached_mutation(async move {
|
||||
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(
|
||||
store.as_ref(),
|
||||
&bucket,
|
||||
guard.as_deref(),
|
||||
Some(store.as_ref()),
|
||||
)
|
||||
.await;
|
||||
invalidate_bucket_usage_snapshot_caches(guard.as_deref(), &bucket).await?;
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(Error::from)?
|
||||
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
|
||||
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
|
||||
result
|
||||
}
|
||||
|
||||
async fn invalidate_bucket_usage_snapshot_caches(
|
||||
@@ -871,9 +849,9 @@ where
|
||||
let primary_seed_epoch = match publication_store {
|
||||
Some(publication_store) => Some(
|
||||
publication_store
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?,
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
@@ -893,9 +871,9 @@ where
|
||||
let backup_seed_epoch = match publication_store {
|
||||
Some(publication_store) => Some(
|
||||
publication_store
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?,
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
@@ -1007,9 +985,9 @@ where
|
||||
let read_epoch = match publication {
|
||||
Some((publication_store, expected_publication_epoch)) => {
|
||||
let epoch = publication_store
|
||||
.data_usage_namespace_cleanup_epoch()
|
||||
.scanner_data_usage_publication_epoch()
|
||||
.await
|
||||
.ok_or_else(|| Error::other("data usage namespace cleanup epoch is unavailable"))?;
|
||||
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
|
||||
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
|
||||
return Err(Error::other("data usage publication epoch changed before snapshot read"));
|
||||
}
|
||||
@@ -1044,8 +1022,9 @@ where
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
|
||||
let publication_guard = match publication {
|
||||
Some((publication_store, expected_publication_epoch)) => {
|
||||
let Some((guard, publication_epoch)) = publication_store.data_usage_namespace_cleanup_guard().await else {
|
||||
return Err(Error::other("data usage namespace cleanup epoch is unavailable"));
|
||||
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
|
||||
else {
|
||||
return Err(Error::other("data usage publication is blocked by data movement"));
|
||||
};
|
||||
if expected_publication_epoch
|
||||
.or(read_epoch)
|
||||
@@ -1057,27 +1036,18 @@ where
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let save_result = match publication {
|
||||
Some((publication_store, _)) => {
|
||||
publication_store
|
||||
.put_usage_snapshot_for_namespace_cleanup(object, revision.as_deref(), &mut put_data, guard)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
store
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut put_data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let save_result = store
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut put_data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
|
||||
Ok(_) => return Ok(()),
|
||||
|
||||
@@ -579,7 +579,6 @@ struct PreparedPartTxnReceipt {
|
||||
dst_meta_path: PathBuf,
|
||||
transaction_path: PathBuf,
|
||||
meta: Bytes,
|
||||
had_old_data: bool,
|
||||
had_old_meta: bool,
|
||||
}
|
||||
|
||||
@@ -587,7 +586,6 @@ struct PublishedPartTxnReceipt {
|
||||
current_data_path: PathBuf,
|
||||
current_meta_path: PathBuf,
|
||||
transaction_path: PathBuf,
|
||||
had_old_data: bool,
|
||||
had_old_meta: bool,
|
||||
}
|
||||
|
||||
@@ -11504,17 +11502,9 @@ impl LocalDisk {
|
||||
admission: Option<multipart_staging::MultipartSourceAdmission>,
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
|
||||
let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
if fuse_with_admission {
|
||||
self.multipart_index.prepare_record_head(path, &buf).await
|
||||
} else {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
}
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
@@ -11530,14 +11520,11 @@ impl LocalDisk {
|
||||
{
|
||||
multipart_source_state::invalidate_active_multipart_source_head(admission.source());
|
||||
}
|
||||
if fuse_with_admission {
|
||||
if admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path) {
|
||||
let precondition = admission.as_ref().map(|a| a.precondition());
|
||||
let root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
|
||||
if let Some(record) = prepared_index {
|
||||
record.run_std()?;
|
||||
}
|
||||
super::fs::create_dir_all_std(&tmp_volume_dir)?;
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&tmp_file_path);
|
||||
@@ -14438,7 +14425,6 @@ impl DiskAPI for LocalDisk {
|
||||
if bucket == RUSTFS_META_MULTIPART_BUCKET && !paths.is_empty() {
|
||||
let mut cached_parts = Vec::with_capacity(paths.len());
|
||||
let mut all_cached = true;
|
||||
let mut verified_source: Option<(&str, PathBuf)> = None;
|
||||
for path_str in paths {
|
||||
let Some(num) = Path::new(path_str)
|
||||
.file_name()
|
||||
@@ -14450,24 +14436,15 @@ impl DiskAPI for LocalDisk {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
let scope_root = multipart_staging::upload_scope_root(path_str, true);
|
||||
let (source, head_already_verified) = match verified_source.as_ref() {
|
||||
Some((prev_root, prev_source)) if *prev_root == scope_root => (prev_source, true),
|
||||
_ => {
|
||||
let Ok(source) = self.io_get_object_open_path(bucket, scope_root) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
verified_source = Some((scope_root, source));
|
||||
(&verified_source.as_ref().expect("just initialized").1, false)
|
||||
}
|
||||
};
|
||||
let Some(part) =
|
||||
multipart_source_state::get_active_multipart_source_part_with_head(source, num, head_already_verified)
|
||||
let Ok(source) = self.io_get_object_open_path(bucket, multipart_staging::upload_scope_root(path_str, true))
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
let Some(part) = multipart_source_state::get_active_multipart_source_part(&source, num) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
cached_parts.push(part);
|
||||
}
|
||||
if all_cached {
|
||||
@@ -14648,7 +14625,6 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(_) => return Ok(Err(DiskError::FileAccessDenied)),
|
||||
Err(err) => return Ok(Err(to_file_error(err).into())),
|
||||
}
|
||||
let mut created_transaction_parent = false;
|
||||
if let Some((integrity, source, destination, expected_len)) = integrity_prep {
|
||||
let stat = match std::fs::symlink_metadata(&source) {
|
||||
Ok(stat) => stat,
|
||||
@@ -14657,7 +14633,6 @@ impl DiskAPI for LocalDisk {
|
||||
if !stat.is_file() || stat.len() != expected_len {
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let presynced_proof = take_presynced_tmp_file(&source);
|
||||
let verify_and_sync = (|| -> std::io::Result<bool> {
|
||||
let mut file = os::open_file_for_sync_std(&source)?;
|
||||
let mut header = [0u8; rustfs_filemeta::shard_integrity::INDEX_HEADER_SIZE];
|
||||
@@ -14665,7 +14640,7 @@ impl DiskAPI for LocalDisk {
|
||||
if header != integrity.index_header() {
|
||||
return Ok(false);
|
||||
}
|
||||
if durability.syncs_data_shards() && !presynced_proof {
|
||||
if durability.syncs_data_shards() && !take_presynced_tmp_file(&source) {
|
||||
file.sync_all()?;
|
||||
}
|
||||
Ok(true)
|
||||
@@ -14675,13 +14650,7 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(false) => return Ok(Err(DiskError::FileCorrupt)),
|
||||
Err(err) => return Ok(Err(to_file_error(err).into())),
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
let has_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
|
||||
let has_hooks = false;
|
||||
let precondition = has_hooks
|
||||
.then(|| admission.as_ref().map(|check| check.precondition()))
|
||||
.flatten();
|
||||
let precondition = admission.as_ref().map(|check| check.precondition());
|
||||
// Publish the immutable generation before preparing the part switch.
|
||||
// Rollback retains the old generation; an unreferenced new index is
|
||||
// reclaimed with the upload directory if preparation is interrupted.
|
||||
@@ -14698,13 +14667,12 @@ impl DiskAPI for LocalDisk {
|
||||
) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
created_transaction_parent = same_parent;
|
||||
}
|
||||
|
||||
let Some(transaction_parent) = transaction_path.parent() else {
|
||||
return Ok(Err(DiskError::InvalidPath));
|
||||
};
|
||||
if !created_transaction_parent && let Err(err) = std::fs::create_dir_all(transaction_parent) {
|
||||
if let Err(err) = std::fs::create_dir_all(transaction_parent) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
|
||||
@@ -14712,35 +14680,17 @@ impl DiskAPI for LocalDisk {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
|
||||
let prepare_result = (|| -> std::io::Result<(bool, bool)> {
|
||||
let old_data_absent = staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT);
|
||||
let old_meta_absent = staging_path.join(PART_TRANSACTION_OLD_META_ABSENT);
|
||||
let prepare_result = (|| -> std::io::Result<bool> {
|
||||
let had_old_data = snapshot_part_transaction_file(
|
||||
&dst_file_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&old_data_absent,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
)?;
|
||||
let had_old_meta = snapshot_part_transaction_file(
|
||||
&dst_meta_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
)?;
|
||||
let had_old_meta = match std::fs::symlink_metadata(&dst_meta_path) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
std::fs::hard_link(&dst_meta_path, staging_path.join(PART_TRANSACTION_OLD_META))?;
|
||||
true
|
||||
}
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "multipart transaction source is a symlink"));
|
||||
}
|
||||
Ok(_) => {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file"));
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
if !had_old_data && std::fs::hard_link(&old_data_absent, &old_meta_absent).is_ok() {
|
||||
false
|
||||
} else {
|
||||
std::fs::write(&old_meta_absent, [])?;
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let new_meta_path = staging_path.join(PART_TRANSACTION_NEW_META);
|
||||
let mut new_meta = std::fs::OpenOptions::new()
|
||||
@@ -14763,11 +14713,11 @@ impl DiskAPI for LocalDisk {
|
||||
if sync_overwrite_backup {
|
||||
os::fsync_dir_std(transaction_parent)?;
|
||||
}
|
||||
Ok((had_old_data, had_old_meta))
|
||||
Ok(had_old_meta)
|
||||
})();
|
||||
|
||||
let (had_old_data, had_old_meta) = match prepare_result {
|
||||
Ok(flags) => flags,
|
||||
let had_old_meta = match prepare_result {
|
||||
Ok(had_old_meta) => had_old_meta,
|
||||
Err(err) => {
|
||||
let _ = remove_dir_all_if_exists(&staging_path);
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
@@ -14783,7 +14733,6 @@ impl DiskAPI for LocalDisk {
|
||||
dst_meta_path,
|
||||
transaction_path: transaction_path.clone(),
|
||||
meta,
|
||||
had_old_data,
|
||||
had_old_meta,
|
||||
},
|
||||
);
|
||||
@@ -14810,25 +14759,23 @@ impl DiskAPI for LocalDisk {
|
||||
.io_get_object_open_path(volume, &txn_rel_path)
|
||||
.ok()
|
||||
.and_then(|txn_open| take_published_part_txn_receipt(&txn_open));
|
||||
let (current_data_path, current_meta_path, transaction_path, skip_obsolete_read_on_commit, cached_old_flags) =
|
||||
match cached_published {
|
||||
Some(entry) => (
|
||||
entry.current_data_path,
|
||||
entry.current_meta_path,
|
||||
entry.transaction_path,
|
||||
!entry.had_old_meta,
|
||||
Some((entry.had_old_data, entry.had_old_meta)),
|
||||
),
|
||||
None => {
|
||||
let (_, current_data_path) = self.io_get_bucket_and_object_path(volume, path)?;
|
||||
let current_meta_path = self.io_get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_path(volume, &txn_rel_path)?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
(current_data_path, current_meta_path, transaction_path, false, None)
|
||||
let (current_data_path, current_meta_path, transaction_path, skip_obsolete_read_on_commit) = match cached_published {
|
||||
Some(entry) => (
|
||||
entry.current_data_path,
|
||||
entry.current_meta_path,
|
||||
entry.transaction_path,
|
||||
!entry.had_old_meta,
|
||||
),
|
||||
None => {
|
||||
let (_, current_data_path) = self.io_get_bucket_and_object_path(volume, path)?;
|
||||
let current_meta_path = self.io_get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.io_get_object_path(volume, &txn_rel_path)?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
};
|
||||
(current_data_path, current_meta_path, transaction_path, false)
|
||||
}
|
||||
};
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let part_number = current_data_path
|
||||
@@ -14927,26 +14874,7 @@ impl DiskAPI for LocalDisk {
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
if let Some((had_old_data, had_old_meta)) = cached_old_flags.filter(|_| action == PartTransactionAction::Commit) {
|
||||
let old_data_name = if had_old_data {
|
||||
PART_TRANSACTION_OLD_DATA
|
||||
} else {
|
||||
PART_TRANSACTION_OLD_DATA_ABSENT
|
||||
};
|
||||
let old_meta_name = if had_old_meta {
|
||||
PART_TRANSACTION_OLD_META
|
||||
} else {
|
||||
PART_TRANSACTION_OLD_META_ABSENT
|
||||
};
|
||||
for name in [PART_TRANSACTION_NEW_META, old_data_name, old_meta_name] {
|
||||
let _ = std::fs::remove_file(cleanup_path.join(name));
|
||||
}
|
||||
if std::fs::remove_dir(&cleanup_path).is_err()
|
||||
&& let Err(err) = remove_dir_all_if_exists(&cleanup_path)
|
||||
{
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
} else if let Err(err) = remove_dir_all_if_exists(&cleanup_path) {
|
||||
if let Err(err) = remove_dir_all_if_exists(&cleanup_path) {
|
||||
return Ok(Err(to_file_error(err).into()));
|
||||
}
|
||||
Ok(Ok(true))
|
||||
@@ -15018,7 +14946,7 @@ impl DiskAPI for LocalDisk {
|
||||
dst_file_path,
|
||||
dst_meta_path,
|
||||
transaction_path,
|
||||
preverified_meta_and_old_flags,
|
||||
preverified_meta_and_old_flag,
|
||||
) = match cached_prepared {
|
||||
Some(entry) => (
|
||||
entry.src_volume_dir,
|
||||
@@ -15027,7 +14955,7 @@ impl DiskAPI for LocalDisk {
|
||||
entry.dst_file_path,
|
||||
Some(entry.dst_meta_path),
|
||||
Some(entry.transaction_path),
|
||||
Some((entry.had_old_data, entry.had_old_meta)),
|
||||
Some(entry.had_old_meta),
|
||||
),
|
||||
None => {
|
||||
let (src_volume_dir, src_file_path) = self.io_get_bucket_and_object_path(src_volume, src_path)?;
|
||||
@@ -15124,7 +15052,7 @@ impl DiskAPI for LocalDisk {
|
||||
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
|
||||
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
|
||||
if !cfg!(test)
|
||||
&& preverified_meta_and_old_flags.is_some()
|
||||
&& preverified_meta_and_old_flag.is_some()
|
||||
&& publish_meta_path.is_file()
|
||||
&& std::fs::metadata(&transaction_meta_path).is_ok_and(|m| m.is_file() && m.len() == meta.len() as u64)
|
||||
{
|
||||
@@ -15168,13 +15096,7 @@ impl DiskAPI for LocalDisk {
|
||||
os::sync_file(&src_file_path).map_err(to_file_error)?;
|
||||
}
|
||||
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
let has_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
|
||||
let has_hooks = false;
|
||||
let precondition = has_hooks
|
||||
.then(|| admission.as_ref().map(|check| check.precondition()))
|
||||
.flatten();
|
||||
let precondition = admission.as_ref().map(|check| check.precondition());
|
||||
|
||||
os::rename_all_with_precondition_and_parent_sync_std(
|
||||
&src_file_path,
|
||||
@@ -15186,19 +15108,17 @@ impl DiskAPI for LocalDisk {
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
if cfg!(test) || src_is_dir {
|
||||
let dst_meta = lstat_std(&dst_file_path).map_err(to_file_error)?;
|
||||
if src_is_dir != dst_meta.is_dir() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "dst_type_changed_after_rename",
|
||||
path = ?dst_file_path,
|
||||
"Disk local rename rejected"
|
||||
);
|
||||
return Err(DiskError::FileAccessDenied.into());
|
||||
}
|
||||
let dst_meta = lstat_std(&dst_file_path).map_err(to_file_error)?;
|
||||
if src_is_dir != dst_meta.is_dir() {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_RENAME_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
reason = "dst_type_changed_after_rename",
|
||||
path = ?dst_file_path,
|
||||
"Disk local rename rejected"
|
||||
);
|
||||
return Err(DiskError::FileAccessDenied.into());
|
||||
}
|
||||
|
||||
if let Some(transaction_publish_meta) = transaction_publish_meta {
|
||||
@@ -15229,8 +15149,8 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let (Some((had_old_data, had_old_meta)), Some(txn_path), Some(meta_path)) =
|
||||
(preverified_meta_and_old_flags, transaction_path.as_ref(), dst_meta_path.as_path().into())
|
||||
if let (Some(had_old_meta), Some(txn_path), Some(meta_path)) =
|
||||
(preverified_meta_and_old_flag, transaction_path.as_ref(), dst_meta_path.as_path().into())
|
||||
{
|
||||
put_published_part_txn_receipt(
|
||||
txn_path,
|
||||
@@ -15238,7 +15158,6 @@ impl DiskAPI for LocalDisk {
|
||||
current_data_path: dst_file_path,
|
||||
current_meta_path: meta_path.to_path_buf(),
|
||||
transaction_path: txn_path.clone(),
|
||||
had_old_data,
|
||||
had_old_meta,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1461,12 +1461,7 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(
|
||||
sync_commit_metadata,
|
||||
!cfg!(test),
|
||||
staging_cleanup,
|
||||
Some(&directory_guard_owned),
|
||||
) {
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, !cfg!(test), staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
|
||||
@@ -1829,9 +1829,7 @@ impl LocalDisk {
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let tx_dir = root.join(transaction.to_string());
|
||||
let _ = take_prepared_delete_receipt(&tx_dir);
|
||||
let _ = take_published_delete_receipt(&tx_dir);
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
@@ -2495,8 +2493,8 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool, bool, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction, destination_missing, old_data_dir_present) =
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
@@ -2531,11 +2529,7 @@ impl LocalDisk {
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
let mut destination_exists = false;
|
||||
let mut old_data_dir_exists = false;
|
||||
let mut scanned_file_path = false;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
scanned_file_path = true;
|
||||
for (visited, entry) in
|
||||
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
|
||||
{
|
||||
@@ -2548,19 +2542,13 @@ impl LocalDisk {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil()) else {
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if id == transaction {
|
||||
destination_exists = true;
|
||||
continue;
|
||||
}
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if old_dir == Some(id) && name_str == id.hyphenated().to_string() {
|
||||
old_data_dir_exists = true;
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
@@ -2619,15 +2607,7 @@ impl LocalDisk {
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((
|
||||
old_dir,
|
||||
target,
|
||||
raw,
|
||||
journal,
|
||||
sole_transaction,
|
||||
scanned_file_path && !destination_exists,
|
||||
scanned_file_path && old_data_dir_exists,
|
||||
)))
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2661,27 +2641,23 @@ impl LocalDisk {
|
||||
if sync {
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
if !destination_missing {
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) => return Err(err),
|
||||
},
|
||||
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
},
|
||||
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
}
|
||||
let old_data_path = old_dir
|
||||
.map(|uuid| file_path.join(uuid.to_string()))
|
||||
.filter(|p| old_data_dir_present || p.is_dir());
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
@@ -2710,15 +2686,8 @@ impl LocalDisk {
|
||||
// A provisional head must stay outside the published recovery transaction.
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let linked = match std::fs::hard_link(&target_in_rollback, &linked_xl) {
|
||||
Ok(()) => true,
|
||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok()
|
||||
}
|
||||
Err(_) => false,
|
||||
};
|
||||
let write_res = if linked {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
@@ -3074,19 +3043,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_original = journal.original_digest.is_some();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = cached_journal_raw.is_none().then(|| journal.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let prune_paths: Vec<_> = if has_target {
|
||||
vec![rollback.clone()]
|
||||
} else {
|
||||
rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
.map(Path::to_path_buf)
|
||||
.collect()
|
||||
};
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
.map(Path::to_path_buf)
|
||||
.collect();
|
||||
|
||||
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
|
||||
let persist_receipt = |finished: bool| -> std::io::Result<()> {
|
||||
@@ -3167,53 +3131,23 @@ impl LocalDisk {
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
let fast_cleanup = cached_journal_raw.is_some();
|
||||
if fast_cleanup {
|
||||
let backup_or_marker = if has_original {
|
||||
Some(STORAGE_FORMAT_FILE_BACKUP)
|
||||
} else if has_target {
|
||||
Some(DELETE_MARKER_ROLLBACK_FILE)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(name) = backup_or_marker
|
||||
&& let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name)))
|
||||
{
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
for name in [PREPARED, FINISHED].into_iter().chain(has_target.then_some(TARGET)) {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
for name in [PREPARED, TARGET, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
for name in [PREPARED, TARGET, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync && let Err(err) = os::fsync_dir_std(&rollback) {
|
||||
|
||||
@@ -429,7 +429,7 @@ impl IndexOwner {
|
||||
os::replace_multipart_source_metadata(&marker, b"1", &self.base, &root)?;
|
||||
os::sync_multipart_owned_record_parents(&self.base.join(".rustfs.sys"), &directory, true, &self.base, &root)?;
|
||||
}
|
||||
Ok(())
|
||||
self.ensure_current_root()
|
||||
}
|
||||
|
||||
fn ensure_current_root(&self) -> io::Result<()> {
|
||||
@@ -547,10 +547,13 @@ impl IndexOwner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
|
||||
let upload = path
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let Some(upload) = path
|
||||
.strip_suffix("/xl.meta")
|
||||
.filter(|upload| canonical_upload(upload).is_some())?;
|
||||
.filter(|upload| canonical_upload(upload).is_some())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let info = match get_file_info(
|
||||
bytes,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
@@ -565,33 +568,10 @@ impl IndexOwner {
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
self.disabled.store(true, Ordering::Release);
|
||||
return None;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let marker = info
|
||||
.metadata
|
||||
.get(crate::disk::multipart_index::RUSTFS_MULTIPART_BUCKET_KEY)
|
||||
.map(|bucket| {
|
||||
self.base
|
||||
.join(INDEX)
|
||||
.join(sha(bucket))
|
||||
.join(upload.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| self.base.join(INDEX));
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
Some(PreparedHeadRecord {
|
||||
owner: Arc::clone(self),
|
||||
upload: upload.to_owned(),
|
||||
info,
|
||||
_lease: lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let Some(prepared) = self.prepare_record_head(path, bytes).await else {
|
||||
return Ok(());
|
||||
};
|
||||
prepared.run().await
|
||||
self.record_info(upload, &info).await
|
||||
}
|
||||
|
||||
async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
|
||||
@@ -713,37 +693,6 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHeadRecord {
|
||||
owner: Arc<IndexOwner>,
|
||||
upload: String,
|
||||
info: FileInfo,
|
||||
_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl PreparedHeadRecord {
|
||||
pub(super) fn run_std(self) -> io::Result<()> {
|
||||
let Self {
|
||||
owner,
|
||||
upload,
|
||||
info,
|
||||
_lease,
|
||||
} = self;
|
||||
match owner.record(&upload, &info) {
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidData => {
|
||||
owner.disabled.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(self) -> Result<()> {
|
||||
let lease = Arc::clone(&self._lease);
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(in crate::disk) async fn wait_for_multipart_index_bootstrap(&self) {
|
||||
|
||||
@@ -428,6 +428,7 @@ impl LocalDisk {
|
||||
let root = store.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, Some(store.lease.clone())).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
permit.validate()?;
|
||||
os::rename_multipart_source_owned_std(&source, &destination, &base, &root, || permit.validate())
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -295,17 +295,10 @@ pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: By
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
if FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) != Some(expected_head_stamp) {
|
||||
return false;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if source.join(STATE).exists()
|
||||
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
|| source.join(".mpu-completions").exists()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
@@ -382,11 +375,7 @@ pub(super) fn finish_active_multipart_part_transaction(
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part_with_head(
|
||||
source: &Path,
|
||||
part_number: usize,
|
||||
head_already_verified: bool,
|
||||
) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
@@ -397,7 +386,7 @@ pub(super) fn get_active_multipart_source_part_with_head(
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
if !head_already_verified && !stamp_matches_active_disk(source, head_stamp) {
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
@@ -620,45 +609,25 @@ impl SourceStore {
|
||||
raw: &[u8],
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
Self::persist_state_with_guard_std(source, base, root, raw, terminal, sync_source_before, None)
|
||||
}
|
||||
fn persist_state_with_guard_std(
|
||||
source: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
raw: &[u8],
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
directory_guard: Option<&os::RenameCommitGuard>,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
let _pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
if sync_source_before {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| IoError::other("empty source directory guard"))?);
|
||||
os::fsync_directory_handle_std(source, Some(&opened))?;
|
||||
}
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
{
|
||||
let opened = std::fs::File::from(
|
||||
_pinned
|
||||
.last()
|
||||
.ok_or_else(|| IoError::other("empty source directory guard"))?
|
||||
.try_clone()?,
|
||||
);
|
||||
os::fsync_directory_handle_std(source, Some(&opened))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
os::fsync_dir_std(source)?;
|
||||
}
|
||||
if let Some(path) = terminal {
|
||||
os::replace_multipart_source_metadata(path, raw, base, root)?;
|
||||
}
|
||||
os::replace_multipart_source_metadata_with_guard(&source.join(STATE), raw, base, root, directory_guard)
|
||||
os::replace_multipart_source_metadata(&source.join(STATE), raw, base, root)
|
||||
}
|
||||
async fn write_owned_backup_and_persist(&self, owner: Uuid, target: bool, bytes: &[u8], state: &SourceState) -> Result<()> {
|
||||
if owner.is_nil() {
|
||||
@@ -755,39 +724,17 @@ impl SourceStore {
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
Self::write_owned_backup_inner_std(source, base, root, owner, target, bytes, sync_source_root, false, true, false)
|
||||
}
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn write_owned_backup_inner_std(
|
||||
source: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
fresh_owner: bool,
|
||||
sync_parents: bool,
|
||||
source_already_pinned: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
} else {
|
||||
original_path(source, owner)
|
||||
};
|
||||
let _pin = if source_already_pinned {
|
||||
None
|
||||
} else {
|
||||
Some(os::pin_multipart_source_directory(source, base, root)?)
|
||||
};
|
||||
let _pin = os::pin_multipart_source_directory(source, base, root)?;
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("source backup has no parent"))?;
|
||||
let _parents = os::mkdir_all_below_existing_base_std(parent, base, root)?;
|
||||
let existed = if fresh_owner {
|
||||
false
|
||||
} else if target {
|
||||
let existed = if target {
|
||||
match os::observe_multipart_source_metadata(&path, base, root, bytes.len() as u64) {
|
||||
Ok((length, sha)) if length == bytes.len() as u64 && sha == raw_sha(bytes) => true,
|
||||
Ok(_) => {
|
||||
@@ -807,10 +754,7 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
if sync_parents {
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)?;
|
||||
}
|
||||
Ok(())
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
@@ -2080,41 +2024,11 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
false,
|
||||
&original_bytes,
|
||||
false,
|
||||
true,
|
||||
old_metadata.is_none(),
|
||||
true,
|
||||
)?;
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
true,
|
||||
previous,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
)?;
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
false,
|
||||
Some(directory_guard),
|
||||
)?;
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -2123,28 +2037,9 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_inner_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
backup_owner,
|
||||
true,
|
||||
previous,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
)?;
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
false,
|
||||
Some(directory_guard),
|
||||
)?;
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
@@ -2374,7 +2269,6 @@ impl NativeMultipartPublication {
|
||||
target_already_synced: bool,
|
||||
fused_in_place: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
directory_guard: Option<&os::RenameCommitGuard>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
@@ -2460,15 +2354,7 @@ impl NativeMultipartPublication {
|
||||
})?;
|
||||
}
|
||||
}
|
||||
SourceStore::persist_state_with_guard_std(
|
||||
source,
|
||||
base,
|
||||
root,
|
||||
&next_state_raw,
|
||||
terminal.as_deref(),
|
||||
!fused_in_place,
|
||||
directory_guard,
|
||||
)?;
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), !fused_in_place)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
@@ -2501,7 +2387,7 @@ impl NativeMultipartPublication {
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, false, staging_cleanup, None)
|
||||
this.finish_on_thread(target_already_synced, false, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -455,11 +455,8 @@ impl LocalDisk {
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part_with_head(
|
||||
&source_object,
|
||||
part.number,
|
||||
from_active_cache,
|
||||
) {
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
|
||||
@@ -611,10 +611,6 @@ pub(crate) mod prepare_rename_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_after_component_opened() -> bool {
|
||||
!AFTER_COMPONENT_OPENED.lock().is_empty()
|
||||
}
|
||||
|
||||
/// Hooks keyed by a directory component name, run before a Windows
|
||||
/// directory pin retries a transient sharing conflict on it.
|
||||
#[cfg(windows)]
|
||||
@@ -3808,7 +3804,7 @@ pub(crate) fn rename_all_with_precondition_and_parent_sync_std(
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run_rename_destination(source, destination);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
if prepared_publication_test_hooks::has_any_hooks() {
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
|
||||
if let Some(check) = precondition {
|
||||
@@ -7202,39 +7198,6 @@ pub(crate) fn pin_multipart_source_directory(
|
||||
}
|
||||
}
|
||||
|
||||
fn same_parent_commit_guard(
|
||||
parent: &Path,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
directory_guard: Option<&RenameCommitGuard>,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(directory_guard) = directory_guard.filter(|g| g.directories.source_path == parent) {
|
||||
let dir = &directory_guard.directories.source;
|
||||
return Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: directory_guard.directories.root.clone(),
|
||||
source_path: parent.to_path_buf(),
|
||||
destination_path: parent.to_path_buf(),
|
||||
source: LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
},
|
||||
destination: std::sync::OnceLock::from(LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
}),
|
||||
destination_base: base.to_path_buf(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = directory_guard;
|
||||
prepare_rename_commit_guard(parent, parent, base, publication_root)
|
||||
}
|
||||
|
||||
/// A small control record always gets its own inode and an atomic replacement.
|
||||
/// The caller owns the upload SourceRW and keeps its namespace lease alive.
|
||||
pub(crate) fn replace_multipart_source_metadata(
|
||||
@@ -7242,21 +7205,11 @@ pub(crate) fn replace_multipart_source_metadata(
|
||||
bytes: &[u8],
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
replace_multipart_source_metadata_with_guard(path, bytes, base, publication_root, None)
|
||||
}
|
||||
|
||||
pub(crate) fn replace_multipart_source_metadata_with_guard(
|
||||
path: &Path,
|
||||
bytes: &[u8],
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
directory_guard: Option<&RenameCommitGuard>,
|
||||
) -> io::Result<()> {
|
||||
let parent = path.parent().ok_or_else(|| io::Error::other("MPU record has no parent"))?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _pinned = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let guard = same_parent_commit_guard(parent, base, publication_root, directory_guard)?;
|
||||
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
|
||||
let staged = parent.join(format!(".mpu-record-{}", uuid::Uuid::new_v4()));
|
||||
let mut prepared = create_prepared_rename_source_with_commit_guard(&staged, path, &guard)?;
|
||||
prepared.write_all(bytes, true)?;
|
||||
@@ -7505,9 +7458,37 @@ pub(crate) fn install_multipart_prepared_head(
|
||||
if original_path.parent() != Some(parent) || original_path == canonical_head {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "MPU proxy must be a private sibling"));
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
let guard = if let Some(directory_guard) = directory_guard.filter(|g| g.directories.source_path == parent) {
|
||||
let dir = &directory_guard.directories.source;
|
||||
RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: directory_guard.directories.root.clone(),
|
||||
source_path: parent.to_path_buf(),
|
||||
destination_path: parent.to_path_buf(),
|
||||
source: LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
},
|
||||
destination: std::sync::OnceLock::from(LinuxCommitDirectory {
|
||||
file: Arc::clone(&dir.file),
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
}),
|
||||
destination_base: base.to_path_buf(),
|
||||
}),
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_guard(parent, parent, base, publication_root)?
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _pinned = pin_multipart_source_directory(parent, base, publication_root)?;
|
||||
let guard = same_parent_commit_guard(parent, base, publication_root, directory_guard)?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let guard = {
|
||||
let _ = directory_guard;
|
||||
prepare_rename_commit_guard(parent, parent, base, publication_root)?
|
||||
};
|
||||
rename_prepared_source_std(&prepared, &original_path, canonical_head, base, publication_root, &guard)?;
|
||||
prepared.path = canonical_head.to_path_buf();
|
||||
fsync_commit_directory_limited_std(parent, &guard)?;
|
||||
@@ -7809,7 +7790,8 @@ pub(crate) fn hardlink_multipart_source_files(
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{AtFlags, FileType, linkat, statat, unlinkat};
|
||||
use rustix::fs::{AtFlags, Mode, OFlags, linkat, openat, unlinkat};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(test)]
|
||||
let source_path = source.to_path_buf();
|
||||
#[cfg(test)]
|
||||
@@ -7824,26 +7806,20 @@ pub(crate) fn hardlink_multipart_source_files(
|
||||
.ok_or_else(|| io::Error::other("missing multipart attempt directory"))?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_hardlink_destination(&source_path, &destination_path);
|
||||
let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
for name in names {
|
||||
if cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let expected = statat(source, name.as_str(), AtFlags::SYMLINK_NOFOLLOW)?;
|
||||
let expected_type = FileType::from_raw_mode(expected.st_mode);
|
||||
if expected_type.is_symlink() {
|
||||
return Err(io::Error::from_raw_os_error(libc::ELOOP));
|
||||
}
|
||||
if !expected_type.is_file() {
|
||||
let original = std::fs::File::from(openat(source, name.as_str(), flags, Mode::empty())?);
|
||||
let expected = original.metadata()?;
|
||||
if !expected.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart shard is not a regular file"));
|
||||
}
|
||||
linkat(source, name.as_str(), destination, name.as_str(), AtFlags::empty())?;
|
||||
let actual = statat(destination, name.as_str(), AtFlags::SYMLINK_NOFOLLOW)?;
|
||||
if !FileType::from_raw_mode(actual.st_mode).is_file()
|
||||
|| actual.st_dev != expected.st_dev
|
||||
|| actual.st_ino != expected.st_ino
|
||||
|| actual.st_size != expected.st_size
|
||||
|| actual.st_mtime != expected.st_mtime
|
||||
{
|
||||
let linked = std::fs::File::from(openat(destination, name.as_str(), flags, Mode::empty())?);
|
||||
let actual = linked.metadata()?;
|
||||
if actual.dev() != expected.dev() || actual.ino() != expected.ino() {
|
||||
let _ = unlinkat(destination, name.as_str(), AtFlags::empty());
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart shard changed during staging"));
|
||||
}
|
||||
@@ -8264,37 +8240,7 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = {
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
|
||||
let owned_base = (base_dir != publication_root.io_path.as_path() && base_dir != publication_root.path.as_path())
|
||||
.then(|| publication_root.open_directory(base_dir).map(std::os::fd::OwnedFd::from))
|
||||
.transpose()?;
|
||||
#[cfg(test)]
|
||||
let has_hook = prepare_rename_test_hooks::has_after_component_opened();
|
||||
#[cfg(not(test))]
|
||||
let has_hook = false;
|
||||
if !has_hook {
|
||||
let base_fd = owned_base
|
||||
.as_ref()
|
||||
.map_or_else(|| publication_root.directory.as_fd(), AsFd::as_fd);
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(Errno::NOENT | Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
match owned_base {
|
||||
Some(base) => base,
|
||||
None => publication_root.open_directory(base_dir)?.into(),
|
||||
}
|
||||
};
|
||||
let base = publication_root.open_directory(base_dir)?.into();
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
// The configured endpoint is trusted; every component beneath it
|
||||
|
||||
@@ -4608,19 +4608,6 @@ fn marker_purge_receipt_path(
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// The first disk slot UUID is assigned when this erasure set is formatted.
|
||||
/// It survives pool reordering and replacement of a disk in that slot.
|
||||
pub fn persistent_set_id(&self) -> Result<Uuid> {
|
||||
self.format
|
||||
.erasure
|
||||
.sets
|
||||
.get(self.set_index)
|
||||
.and_then(|disks| disks.first())
|
||||
.copied()
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Error::other("erasure set has no persistent format identity"))
|
||||
}
|
||||
|
||||
/// Persist a quorum DELETE receipt outside the object marker itself. The
|
||||
/// receipt is written to the metadata bucket so it survives a node restart
|
||||
/// while an inaccessible member still holds the stale marker.
|
||||
|
||||
@@ -443,7 +443,6 @@ impl ECStore {
|
||||
},
|
||||
other => StorageError::Lock(other),
|
||||
})?;
|
||||
let ns_guard = Arc::new(ns_guard);
|
||||
|
||||
let recover = Box::pin(async {
|
||||
self.ensure_bucket_volume_absent_on_all_sets(bucket).await?;
|
||||
@@ -931,16 +930,15 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) -> Result<()> {
|
||||
let namespace_guard = guard.map(Arc::as_ref);
|
||||
run_bucket_usage_cleanup(namespace_guard, bucket, async {
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, namespace_guard).await?;
|
||||
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
|
||||
run_bucket_usage_cleanup(guard, bucket, async {
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
|
||||
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) {
|
||||
async fn cleanup_bucket_usage_best_effort(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) {
|
||||
if let Err(err) = self.cleanup_bucket_usage(bucket, guard).await {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
@@ -1031,7 +1029,6 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = ns_guard.map(Arc::new);
|
||||
|
||||
let existing_bucket_info = match self.get_bucket_info_from_sets(bucket, &BucketOptions::default()).await {
|
||||
Ok(info) => Some(info),
|
||||
@@ -1149,7 +1146,7 @@ impl ECStore {
|
||||
metadata_sys::set_new_bucket_metadata_intent_in(&self.ctx, meta.clone()).await?;
|
||||
if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}")));
|
||||
}
|
||||
@@ -1165,7 +1162,7 @@ impl ECStore {
|
||||
|
||||
if let Err(err) = await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"physical bucket creation",
|
||||
await_bucket_namespace_operation(
|
||||
@@ -1194,9 +1191,9 @@ impl ECStore {
|
||||
{
|
||||
warn!("best-effort bucket heal after BucketExists failed: {heal_err}");
|
||||
}
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_deref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
if confirmed_missing && !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) {
|
||||
error!("make bucket failed: {err}");
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_deref())
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
@@ -1217,10 +1214,10 @@ impl ECStore {
|
||||
.await;
|
||||
let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| ns_guard.as_deref().is_some_and(|guard| guard.is_lock_lost());
|
||||
|| ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
|
||||
if let Err(err) = metadata_result {
|
||||
if !metadata_lock_lost {
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_deref())
|
||||
self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref())
|
||||
.await;
|
||||
}
|
||||
return Err(err);
|
||||
@@ -1583,7 +1580,6 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let ns_guard = ns_guard.map(Arc::new);
|
||||
|
||||
let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
|
||||
let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
|
||||
@@ -1619,7 +1615,7 @@ impl ECStore {
|
||||
crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_with_guard_in(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
Some(ns_guard.as_deref().unwrap_or(&guard)),
|
||||
Some(ns_guard.as_ref().unwrap_or(&guard)),
|
||||
)
|
||||
.await?;
|
||||
match state {
|
||||
@@ -1645,7 +1641,7 @@ impl ECStore {
|
||||
if sr_mark_delete {
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket delete marker creation",
|
||||
await_bucket_namespace_operation(
|
||||
@@ -1684,7 +1680,7 @@ impl ECStore {
|
||||
object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
|
||||
bucket,
|
||||
"Object Lock configuration fence during bucket deletion",
|
||||
run_physical_bucket_deletion(ns_guard.as_deref(), bucket, async {
|
||||
run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
|
||||
self.delete_bucket_on_sets(bucket, &delete_opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket]))
|
||||
@@ -1713,12 +1709,12 @@ impl ECStore {
|
||||
if let Some(guard) = bucket_lifecycle_guard.as_ref() {
|
||||
record_opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
if let Some(guard) = ns_guard.as_deref() {
|
||||
if let Some(guard) = ns_guard.as_ref() {
|
||||
record_opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket retirement publication",
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
|
||||
@@ -1728,7 +1724,7 @@ impl ECStore {
|
||||
// V1 ledger must not block quota admission after same-name recreation.
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_deref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"retired bucket quota ledger cleanup",
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
|
||||
@@ -1739,7 +1735,7 @@ impl ECStore {
|
||||
self.cleanup_bucket_usage_best_effort(bucket, ns_guard.as_ref()).await;
|
||||
|
||||
if let Err(err) = self
|
||||
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_deref())
|
||||
.cleanup_deleted_bucket_metadata(bucket, sr_purge, ns_guard.as_ref())
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
|
||||
@@ -1120,20 +1120,6 @@ impl ECStore {
|
||||
Some(epoch)
|
||||
}
|
||||
|
||||
/// Namespace cleanup edits an existing usage baseline; it does not certify
|
||||
/// a scan. Retain the movement fence without requiring scanning to resume.
|
||||
pub(crate) async fn data_usage_namespace_cleanup_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
|
||||
let operation_guard = self.ctx.data_movement_operation_gate().read_owned().await;
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
return None;
|
||||
}
|
||||
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
|
||||
}
|
||||
|
||||
pub(crate) async fn data_usage_namespace_cleanup_epoch(&self) -> Option<u64> {
|
||||
self.data_usage_namespace_cleanup_guard().await.map(|(_, epoch)| epoch)
|
||||
}
|
||||
|
||||
/// Acquire a storage-owned read admission for a coordinator's final
|
||||
/// scanner publication. The guard remains in the context's lease table,
|
||||
/// so a local movement writer cannot pass the peer while its authoritative
|
||||
|
||||
@@ -4937,87 +4937,6 @@ impl ECStore {
|
||||
Ok((idx, result))
|
||||
}
|
||||
|
||||
/// Compare the logical revision across all pools before publishing a
|
||||
/// namespace-only usage update to a writable pool. A suspended source is
|
||||
/// retained for decommission to reconcile against the newer target.
|
||||
pub(crate) async fn put_usage_snapshot_for_namespace_cleanup(
|
||||
&self,
|
||||
object: &str,
|
||||
expected_revision: Option<&str>,
|
||||
data: &mut PutObjReader,
|
||||
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<ObjectInfo> {
|
||||
// The caller owns the bucket namespace and movement guards. Acquire
|
||||
// the fixed object domain, then every physical domain in pool order.
|
||||
let guards = self
|
||||
.acquire_all_physical_object_write_locks("usage_namespace_cleanup", RUSTFS_META_BUCKET, object)
|
||||
.await?;
|
||||
let mut opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
};
|
||||
for guard in &guards {
|
||||
guard.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(guard) = namespace_guard {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let source = match self
|
||||
.get_latest_object_info_for_conditional_write(RUSTFS_META_BUCKET, object, &opts)
|
||||
.await
|
||||
.and_then(|(info, idx)| resolve_latest_object_access(RUSTFS_META_BUCKET, object, info, idx, &opts))
|
||||
{
|
||||
Ok((info, _)) => Some(info),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if source.as_ref().and_then(|info| info.etag.as_deref()) != expected_revision {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if source.as_ref().is_some_and(|info| {
|
||||
info.version_id.is_some_and(|version| !version.is_nil()) || info.etag.as_ref().is_none_or(String::is_empty)
|
||||
}) {
|
||||
return Err(StorageError::other("usage cleanup requires an unversioned snapshot with a revision"));
|
||||
}
|
||||
let target = self.get_pool_idx_no_lock(RUSTFS_META_BUCKET, object, data.size()).await?;
|
||||
let target_info = match self.pools[target].get_object_info(RUSTFS_META_BUCKET, object, &opts).await {
|
||||
Ok(info) => Some(info),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
opts.http_preconditions = Some(match target_info.as_ref() {
|
||||
Some(info) => crate::storage_api_contracts::object::HTTPPreconditions {
|
||||
if_match: Some(
|
||||
info.etag
|
||||
.as_ref()
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.ok_or_else(|| StorageError::other("usage cleanup target has no revision"))?,
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
None => crate::storage_api_contracts::object::HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
});
|
||||
let latest = source
|
||||
.iter()
|
||||
.chain(target_info.iter())
|
||||
.filter_map(|info| info.mod_time)
|
||||
.fold(OffsetDateTime::now_utc(), std::cmp::max);
|
||||
opts.mod_time = Some(
|
||||
latest
|
||||
.checked_add(time::Duration::nanoseconds(1))
|
||||
.ok_or_else(|| StorageError::other("usage cleanup snapshot timestamp is exhausted"))?,
|
||||
);
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
self.pools[target].put_object(RUSTFS_META_BUCKET, object, data, &opts).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
pub(super) async fn handle_put_object(
|
||||
@@ -7086,7 +7005,7 @@ mod tests {
|
||||
};
|
||||
use crate::set_disk::same_distributed_lock_domain;
|
||||
use crate::set_disk::{SetDisks, disk_call_counters};
|
||||
use crate::storage_api_contracts::bucket::{BucketOptions, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use crate::storage_api_contracts::list::ListOperations as _;
|
||||
use bytes::Bytes;
|
||||
@@ -10428,296 +10347,6 @@ mod tests {
|
||||
multipool_delete_test_store(bucket, true).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_fresh_bucket_after_canceled_decommission() {
|
||||
let (_dirs, store) = multipool_version_test_store("canceled-decommission-existing").await;
|
||||
let mut canceled = prepared_pool_test_status(1, false);
|
||||
canceled.decommission = Some(PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: vec![prepared_pool_test_status(0, false), canceled],
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.handle_make_bucket("canceled-decommission-fresh", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("canceled decommission must not block creation of a new bucket");
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
"scanner publication must remain fenced"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_bucket_after_failed_decommission() {
|
||||
let (_dirs, store) = multipool_version_test_store("decommission-existing-bucket").await;
|
||||
let bucket = "decommission-new-bucket";
|
||||
let snapshot_time = std::time::SystemTime::now() - Duration::from_secs(60);
|
||||
let mut snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(snapshot_time),
|
||||
scanner_cycle: Some(37),
|
||||
scanner_epoch: Some(2),
|
||||
buckets_count: 1,
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.to_string(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 42,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.to_string(), 42);
|
||||
snapshot.calculate_totals();
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("store a stale bucket-usage baseline before the failed decommission");
|
||||
let (old_snapshot, usage_owner) = store
|
||||
.get_latest_accessible_object_info_with_idx(
|
||||
RUSTFS_META_BUCKET,
|
||||
crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("identify the pool holding the old usage baseline");
|
||||
let mut pool_statuses = vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, false)];
|
||||
pool_statuses[usage_owner].decommission = Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: pool_statuses,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("a failed decommission must not block creation of an unrelated bucket");
|
||||
store
|
||||
.handle_get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("the created bucket must be visible");
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("load the cleaned usage baseline");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket), "the new bucket must not inherit old usage");
|
||||
assert_eq!(usage.scanner_cycle, Some(37), "namespace cleanup must not certify a new scan");
|
||||
assert_eq!(usage.scanner_epoch, Some(2), "namespace cleanup must preserve the scanner authority");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (_, current_owner) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the new snapshot authority");
|
||||
assert_ne!(current_owner, usage_owner, "cleanup must publish to a writable pool");
|
||||
let retained_source = store.pools[usage_owner]
|
||||
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("decommission retains its original source for reconciliation");
|
||||
assert_eq!(retained_source.etag, old_snapshot.etag);
|
||||
let stale_update = store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
old_snapshot.etag.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"stale snapshot".to_vec()),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(stale_update, Err(StorageError::PreconditionFailed)));
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("a stale source CAS must not replace the writable target");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket));
|
||||
assert!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
"scanner publication must remain fenced"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canceled_create_retains_usage_cleanup_namespace_owner() {
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-cancellation-existing").await;
|
||||
let bucket = "cleanup-cancellation-new";
|
||||
let mut snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.to_owned(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 42,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.to_owned(), 42);
|
||||
snapshot.calculate_totals();
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the stale namespace usage");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (_, source) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("find the source snapshot");
|
||||
let mut pools = vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, false)];
|
||||
pools[source].decommission = Some(PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
});
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools,
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::AfterNamespace);
|
||||
let create_store = store.clone();
|
||||
let create = tokio::spawn(async move { create_store.handle_make_bucket(bucket, &MakeBucketOptions::default()).await });
|
||||
tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("cleanup reaches its physical commit with the namespace held");
|
||||
create.abort();
|
||||
assert!(create.await.expect_err("the request waiter was canceled").is_cancelled());
|
||||
let namespace = store
|
||||
.new_ns_lock(bucket, bucket)
|
||||
.await
|
||||
.expect("open the bucket namespace lock");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), namespace.get_write_lock(Duration::from_secs(1)))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancellation must not release the namespace before its cleanup finishes"
|
||||
);
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
tokio::time::timeout(Duration::from_secs(10), store.handle_make_bucket(bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("the resumed create must not remain blocked")
|
||||
.expect("a new request resumes after the old cleanup releases its namespace");
|
||||
let usage = crate::data_usage::load_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.expect("read the cleaned baseline");
|
||||
assert!(!usage.buckets_usage.contains_key(bucket));
|
||||
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_cleanup_rejects_an_unreadable_pool() {
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-unreadable-pool").await;
|
||||
let snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the usage baseline");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (before, source) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("find the baseline owner");
|
||||
let unavailable = 1 - source;
|
||||
for disk in store.pools[unavailable]
|
||||
.get_disks_by_key(object)
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
{
|
||||
disk.close().await.expect("stop disk health monitoring");
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
let update = store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
before.etag.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"unproven authority".to_vec()),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(update.is_err(), "an unreadable pool may hold a newer snapshot");
|
||||
let after = store.pools[source]
|
||||
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the readable baseline remains intact");
|
||||
assert_eq!(after.etag, before.etag);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_cleanup_rejects_namespace_loss_before_physical_commit() {
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
|
||||
|
||||
let (_dirs, store) = multipool_version_test_store("cleanup-fence-existing").await;
|
||||
let snapshot = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
crate::data_usage::store_data_usage_in_backend(snapshot, store.clone())
|
||||
.await
|
||||
.expect("seed the protected baseline");
|
||||
let object = crate::data_usage::DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let (before, _) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the baseline revision");
|
||||
let ttl = Duration::from_secs(60);
|
||||
let namespace = NamespaceLock::new("usage-cleanup-commit-fence".to_owned(), Arc::new(LocalClient::new()));
|
||||
let request = LockRequest::new(ObjectKey::new("cleanup-fence-existing", ""), LockType::Exclusive, "usage-cleanup")
|
||||
.with_acquire_timeout(Duration::from_secs(1))
|
||||
.with_ttl(ttl)
|
||||
.with_refresh_interval(ttl);
|
||||
let guard = Arc::new(
|
||||
namespace
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("acquire the namespace lease")
|
||||
.expect("namespace lease is available"),
|
||||
);
|
||||
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, object, PutObjectCommitPause::AfterNamespace);
|
||||
let update_store = store.clone();
|
||||
let update_guard = guard.clone();
|
||||
let revision = before.etag.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
let _movement = update_store
|
||||
.data_usage_namespace_cleanup_guard()
|
||||
.await
|
||||
.expect("admit the namespace-only update");
|
||||
update_store
|
||||
.put_usage_snapshot_for_namespace_cleanup(
|
||||
object,
|
||||
revision.as_deref(),
|
||||
&mut PutObjReader::from_vec(b"must not commit".to_vec()),
|
||||
Some(update_guard.as_ref()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("the update reaches its commit barrier");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(ttl + Duration::from_millis(1)).await;
|
||||
assert!(guard.is_lock_lost(), "the parent namespace lease has expired");
|
||||
tokio::time::resume();
|
||||
barrier.release();
|
||||
assert!(update.await.expect("the cleanup task completes").is_err());
|
||||
let (after, _) = store
|
||||
.get_latest_accessible_object_info_with_idx(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the original baseline remains readable");
|
||||
assert_eq!(after.etag, before.etag, "a lost namespace lease must fence the physical write");
|
||||
}
|
||||
|
||||
async fn multipool_delete_test_store(bucket: &str, versioned: bool) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
|
||||
@@ -690,19 +690,6 @@ pub(super) enum ScannerPauseBacklogAttemptDecision {
|
||||
pub struct ScannerPauseBacklogReplicaId {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
/// Stable format identity; indices are only the current routing location.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub set_id: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogReplicaId {
|
||||
fn from_set(set: &SetDisks) -> Result<Self, EcstoreError> {
|
||||
Ok(Self {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
set_id: Some(set.persistent_set_id()?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
|
||||
@@ -719,23 +706,9 @@ impl ScannerPauseBacklogCommitRecord {
|
||||
|
||||
fn validate(&self) -> Result<(), String> {
|
||||
self.ledger.validate()?;
|
||||
if self.replicas.is_empty()
|
||||
|| !self
|
||||
.replicas
|
||||
.windows(2)
|
||||
.all(|pair| (pair[0].pool_index, pair[0].set_index) < (pair[1].pool_index, pair[1].set_index))
|
||||
{
|
||||
if self.replicas.is_empty() || !self.replicas.windows(2).all(|pair| pair[0] < pair[1]) {
|
||||
return Err("scanner pause backlog commit has invalid replica membership".to_string());
|
||||
}
|
||||
let mut identities = BTreeSet::new();
|
||||
if self
|
||||
.replicas
|
||||
.iter()
|
||||
.filter_map(|id| id.set_id)
|
||||
.any(|id| id.is_nil() || !identities.insert(id))
|
||||
{
|
||||
return Err("scanner pause backlog commit has invalid persistent set identities".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -774,10 +747,6 @@ impl ScannerPauseBacklogReplicaRecord {
|
||||
}
|
||||
if let Some(committed) = &self.committed {
|
||||
committed.validate()?;
|
||||
if committed.replicas.iter().any(|id| id.set_id.is_some()) && committed.replicas.iter().any(|id| id.set_id.is_none())
|
||||
{
|
||||
return Err("scanner pause backlog membership mixes positional and persistent identities".to_string());
|
||||
}
|
||||
if let Some(stable) = &self.stable {
|
||||
let stable_key = (stable.writer_epoch, stable.generation);
|
||||
let committed_key = (committed.ledger.writer_epoch, committed.ledger.generation);
|
||||
@@ -938,19 +907,9 @@ fn unix_now() -> u64 {
|
||||
}
|
||||
|
||||
async fn read_scanner_pause_backlog_replica(store: Arc<SetDisks>) -> ScannerPauseBacklogReplica {
|
||||
let id = match ScannerPauseBacklogReplicaId::from_set(&store) {
|
||||
Ok(id) => id,
|
||||
Err(error) => {
|
||||
return ScannerPauseBacklogReplica {
|
||||
id: ScannerPauseBacklogReplicaId {
|
||||
pool_index: store.pool_index,
|
||||
set_index: store.set_index,
|
||||
set_id: None,
|
||||
},
|
||||
revision: None,
|
||||
state: ScannerPauseBacklogReplicaState::Unavailable(error.to_string()),
|
||||
};
|
||||
}
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: store.pool_index,
|
||||
set_index: store.set_index,
|
||||
};
|
||||
let reader = match store
|
||||
.get_object_reader(
|
||||
@@ -1374,48 +1333,8 @@ fn select_scanner_pause_backlog_roll_forward(replicas: &[ScannerPauseBacklogRepl
|
||||
.then(|| newest.clone())
|
||||
}
|
||||
|
||||
/// Resolve stored membership against immutable set identities before checking
|
||||
/// commit proofs. A missing identity is never rebound to a new set at its old
|
||||
/// index. Legacy records retain their positional interpretation until an
|
||||
/// existing commit or rollback proof permits publishing the bound membership.
|
||||
fn normalize_scanner_pause_backlog_membership(
|
||||
replicas: &mut [ScannerPauseBacklogReplica],
|
||||
) -> Result<(), ScannerPauseBacklogSelectionError> {
|
||||
let mut by_location = HashMap::with_capacity(replicas.len());
|
||||
let mut by_identity = HashMap::with_capacity(replicas.len());
|
||||
for replica in replicas.iter() {
|
||||
let id = replica.id;
|
||||
if by_location.insert((id.pool_index, id.set_index), id).is_some()
|
||||
|| id
|
||||
.set_id
|
||||
.is_some_and(|identity| identity.is_nil() || by_identity.insert(identity, id).is_some())
|
||||
{
|
||||
return Err(ScannerPauseBacklogSelectionError::AuthorityConflict(
|
||||
"scanner pause backlog has duplicate or invalid storage identities".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
for replica in replicas {
|
||||
if let ScannerPauseBacklogReplicaState::Valid(record) = &mut replica.state
|
||||
&& let Some(commit) = record.committed.as_mut()
|
||||
{
|
||||
for member in &mut commit.replicas {
|
||||
let current = match member.set_id {
|
||||
Some(identity) => by_identity.get(&identity),
|
||||
None => by_location.get(&(member.pool_index, member.set_index)),
|
||||
};
|
||||
if let Some(current) = current {
|
||||
*member = *current;
|
||||
}
|
||||
}
|
||||
commit.replicas.sort_unstable();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn select_scanner_pause_backlog_replicas(
|
||||
mut replicas: Vec<ScannerPauseBacklogReplica>,
|
||||
replicas: Vec<ScannerPauseBacklogReplica>,
|
||||
) -> Result<LoadedScannerPauseBacklog, ScannerPauseBacklogSelectionFailure> {
|
||||
if replicas.is_empty() {
|
||||
return Err(scanner_pause_backlog_selection_failure(
|
||||
@@ -1435,9 +1354,6 @@ fn select_scanner_pause_backlog_replicas(
|
||||
)),
|
||||
));
|
||||
}
|
||||
if let Err(error) = normalize_scanner_pause_backlog_membership(&mut replicas) {
|
||||
return Err(scanner_pause_backlog_selection_failure(replicas, error));
|
||||
}
|
||||
|
||||
let replica_ids = scanner_pause_backlog_replica_ids(&replicas);
|
||||
let authoritative_commit = match select_scanner_pause_backlog_commit(&replicas, &replica_ids) {
|
||||
@@ -1596,13 +1512,12 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
) -> Result<(Vec<ScannerPauseBacklogReplica>, Vec<ScannerPauseBacklogReplica>), ScannerPauseBacklogRetirementError> {
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut decoded = Vec::with_capacity(replicas.len());
|
||||
let mut writable_ids = BTreeSet::new();
|
||||
let mut surviving = Vec::new();
|
||||
let mut has_source = false;
|
||||
for replica in replicas {
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: replica.pool_index,
|
||||
set_index: replica.set_index,
|
||||
set_id: replica.set_id,
|
||||
};
|
||||
if !ids.insert(id) {
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
@@ -1633,7 +1548,7 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
state,
|
||||
};
|
||||
if replica.pool_index != source_pool_index && replica.writable {
|
||||
writable_ids.insert(id);
|
||||
surviving.push(decoded_replica.clone());
|
||||
}
|
||||
decoded.push(decoded_replica);
|
||||
}
|
||||
@@ -1642,12 +1557,6 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
"scanner pause backlog retirement has no native source record",
|
||||
));
|
||||
}
|
||||
normalize_scanner_pause_backlog_membership(&mut decoded).map_err(scanner_pause_backlog_retirement_selection_error)?;
|
||||
let surviving = decoded
|
||||
.iter()
|
||||
.filter(|replica| writable_ids.contains(&replica.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
Ok((decoded, surviving))
|
||||
}
|
||||
|
||||
@@ -1965,9 +1874,11 @@ where
|
||||
let loaded_ids = loaded.replicas.iter().map(|replica| replica.id).collect::<BTreeSet<_>>();
|
||||
let writable_ids = writable
|
||||
.iter()
|
||||
.map(|set| ScannerPauseBacklogReplicaId::from_set(set))
|
||||
.collect::<Result<BTreeSet<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
.map(|set| ScannerPauseBacklogReplicaId {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
})
|
||||
.collect::<BTreeSet<_>>();
|
||||
if writable_ids != loaded_ids {
|
||||
return Err("scanner pause backlog replica topology changed during commit".to_string());
|
||||
}
|
||||
@@ -1990,7 +1901,10 @@ where
|
||||
let results = tokio::spawn(async move {
|
||||
let mut results = Vec::with_capacity(writable.len());
|
||||
for set in writable {
|
||||
let id = ScannerPauseBacklogReplicaId::from_set(&set).map_err(|error| error.to_string())?;
|
||||
let id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
};
|
||||
if only.as_ref().is_some_and(|only| !only.contains(&id)) {
|
||||
continue;
|
||||
}
|
||||
@@ -2014,10 +1928,10 @@ where
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok::<_, String>(results)
|
||||
results
|
||||
})
|
||||
.await
|
||||
.map_err(|err| format!("scanner pause backlog publication owner failed: {err}"))??;
|
||||
.map_err(|err| format!("scanner pause backlog publication owner failed: {err}"))?;
|
||||
|
||||
let failures = results
|
||||
.iter()
|
||||
@@ -2530,142 +2444,6 @@ mod tests {
|
||||
(root, store)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority() {
|
||||
assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_pool_reindex_with_failed_membership_publication_preserves_authority() {
|
||||
assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(true);
|
||||
}
|
||||
|
||||
fn assert_native_legacy_upgrade_and_pool_reindex_preserve_backlog_authority(fail_publication: bool) {
|
||||
run_native_retirement_test(async move || {
|
||||
use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault;
|
||||
|
||||
let (root, store) = native_pause_backlog_store(3, 1).await;
|
||||
let now = unix_now();
|
||||
let original = seed_native_expansion_ledger(&store, now).await;
|
||||
// Model bytes written by the previous positional-membership writer.
|
||||
for id in native_replica_ids(&store) {
|
||||
let (bytes, _) = native_replica_bytes(&store.pools[id.pool_index].disk_set[0]).await;
|
||||
let ScannerPauseBacklogReplicaState::Valid(mut record) = decode_scanner_pause_backlog_ledger(&bytes) else {
|
||||
panic!("native seed is valid");
|
||||
};
|
||||
record.replica_schema_version = 1;
|
||||
for member in &mut record.committed.as_mut().expect("seed commit").replicas {
|
||||
member.set_id = None;
|
||||
}
|
||||
record.validate().expect("previous schema fixture");
|
||||
replace_native_pause_backlog_replica(&store, id, *record).await;
|
||||
}
|
||||
let fault =
|
||||
NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&store.pools[1].disk_set[0]), "publish", 1);
|
||||
assert!(
|
||||
ScannerPauseBacklogController::claim(Arc::clone(&store), now + 60)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(fault);
|
||||
let interrupted = load_scanner_pause_backlog(Arc::clone(&store))
|
||||
.await
|
||||
.expect("mixed legacy and bound records retain the rollback proof");
|
||||
assert_eq!(interrupted.ledger, original);
|
||||
let upgraded = ScannerPauseBacklogController::claim(Arc::clone(&store), now + 61)
|
||||
.await
|
||||
.expect("retry finishes the format upgrade");
|
||||
let expected = upgraded.loaded.ledger.clone();
|
||||
drop(upgraded);
|
||||
for pool in &store.pools {
|
||||
let (bytes, _) = native_replica_bytes(&pool.disk_set[0]).await;
|
||||
let ScannerPauseBacklogReplicaState::Valid(record) = decode_scanner_pause_backlog_ledger(&bytes) else {
|
||||
panic!("upgraded native record is valid");
|
||||
};
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
assert!(
|
||||
record
|
||||
.committed
|
||||
.expect("upgraded membership")
|
||||
.replicas
|
||||
.iter()
|
||||
.all(|id| id.set_id.is_some())
|
||||
);
|
||||
}
|
||||
let surviving_id = native_replica_id(&store, 2, 0).set_id;
|
||||
let (source, _) = native_replica_bytes(&store.pools[1].disk_set[0]).await;
|
||||
store
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(1, source.len() * 2)
|
||||
.await
|
||||
.expect("start retiring the middle pool");
|
||||
store
|
||||
.retire_scanner_pause_backlog_for_test(1, 0)
|
||||
.await
|
||||
.expect("hand off the native backlog");
|
||||
store
|
||||
.complete_scanner_pause_backlog_retirement_for_test(1)
|
||||
.await
|
||||
.expect("persist the completed middle pool through the ILM receipt checks");
|
||||
shutdown_native_retirement_store(store).await;
|
||||
|
||||
let reordered = super::super::tests::setup_scanner_cycle_store_with_pool_order(
|
||||
root.path(),
|
||||
false,
|
||||
&[0, 2, 3],
|
||||
1,
|
||||
NATIVE_RETIREMENT_DRIVES_PER_SET,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(native_replica_id(&reordered, 1, 0).set_id, surviving_id);
|
||||
assert_ne!(native_replica_id(&reordered, 2, 0).set_id, surviving_id);
|
||||
let loaded = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("removing a middle pool and appending a new one preserves native authority");
|
||||
assert_eq!(loaded.ledger, expected);
|
||||
assert_eq!(loaded.persistence_state, "membership_repair_pending");
|
||||
if fail_publication {
|
||||
// Match the reported partial publication: the two existing
|
||||
// pools reject the new commit while the replacement accepts it.
|
||||
let faults = [0, 1].map(|pool| {
|
||||
NativeScannerPauseBacklogWriteFault::fail_before_write(
|
||||
Arc::clone(&reordered.pools[pool].disk_set[0]),
|
||||
"publish",
|
||||
2,
|
||||
)
|
||||
});
|
||||
let error = ScannerPauseBacklogController::claim(Arc::clone(&reordered), now + 120)
|
||||
.await
|
||||
.err()
|
||||
.expect("the incomplete publication must fail");
|
||||
assert!(error.contains("pool 0 set 0") && error.contains("pool 1 set 0"), "{error}");
|
||||
drop(faults);
|
||||
let recovered = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("partial publication after reindexing retains proven authority");
|
||||
assert_eq!(recovered.ledger, expected);
|
||||
assert!(recovered.durable);
|
||||
}
|
||||
let (source, _) = native_replica_bytes(&reordered.pools[1].disk_set[0]).await;
|
||||
reordered
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(1, source.len() * 2)
|
||||
.await
|
||||
.expect("retire the reindexed member before a scanner writer completes the membership change");
|
||||
reordered
|
||||
.retire_scanner_pause_backlog_for_test(1, 0)
|
||||
.await
|
||||
.expect("retirement uses the same bound authority");
|
||||
let surviving = load_scanner_pause_backlog(Arc::clone(&reordered))
|
||||
.await
|
||||
.expect("remaining members retain the complete ledger");
|
||||
assert_eq!(surviving.ledger, expected);
|
||||
assert!(surviving.all_current_replicas_stable());
|
||||
shutdown_native_retirement_store(reordered).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_single_pool_partial_commit_converges_across_24_sets() {
|
||||
@@ -2814,7 +2592,9 @@ mod tests {
|
||||
);
|
||||
|
||||
let stable = durable_ledger(unix_now());
|
||||
let members = native_replica_ids(&store);
|
||||
let members = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let committed = ScannerPauseBacklogCommitRecord::new(stable.clone(), members);
|
||||
let record = ScannerPauseBacklogReplicaRecord::new(Some(stable), Some(committed));
|
||||
let bytes = encode_scanner_pause_backlog_record(&record).expect("stable replica should encode");
|
||||
@@ -3241,7 +3021,10 @@ mod tests {
|
||||
let now = unix_now().saturating_add(60);
|
||||
let replacement = claim_scanner_pause_backlog_writer(&original, now).expect("next writer generation");
|
||||
let new_commit = ScannerPauseBacklogCommitRecord::new(replacement.clone(), current_ids.clone());
|
||||
let failed_id = native_replica_id(&store, failed_pool, 0);
|
||||
let failed_id = ScannerPauseBacklogReplicaId {
|
||||
pool_index: failed_pool,
|
||||
set_index: 0,
|
||||
};
|
||||
let fault = NativeScannerPauseBacklogWriteFault::fail_before_write(
|
||||
Arc::clone(&store.pools[failed_pool].disk_set[0]),
|
||||
"publish",
|
||||
@@ -3770,8 +3553,10 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.expect("old cohort commit");
|
||||
let now = unix_now().saturating_add(60);
|
||||
let ids = native_replica_ids(&store);
|
||||
let newer_member = native_replica_id(&store, 2, 1);
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let newer_member = replica_id(2, 1);
|
||||
let newer = claim_scanner_pause_backlog_writer(&original, now).expect("newer generation");
|
||||
let newer_commit = ScannerPauseBacklogCommitRecord::new(newer.clone(), ids.clone());
|
||||
for id in &ids {
|
||||
@@ -3832,12 +3617,14 @@ mod tests {
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let wedge = native_interleaved_expansion_wedge(&store, &original, now).await;
|
||||
let ids = native_replica_ids(&store);
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
// The retiring source holds the newest stable ledger.
|
||||
let newest = claim_scanner_pause_backlog_writer(&wedge.acknowledged, now + 1).expect("newest generation");
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
native_replica_id(&store, 0, 0),
|
||||
replica_id(0, 0),
|
||||
ScannerPauseBacklogReplicaRecord::new(
|
||||
Some(newest.clone()),
|
||||
Some(ScannerPauseBacklogCommitRecord::new(newest.clone(), ids)),
|
||||
@@ -3888,8 +3675,10 @@ mod tests {
|
||||
set_runtime_error(None);
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let ids = native_replica_ids(&store);
|
||||
let newer_member = native_replica_id(&store, 2, 1);
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
let newer_member = replica_id(2, 1);
|
||||
let others = ids.iter().copied().filter(|id| *id != newer_member).collect::<Vec<_>>();
|
||||
let middle = claim_scanner_pause_backlog_writer(&original, now).expect("middle generation");
|
||||
let newest = claim_scanner_pause_backlog_writer(&middle, now + 1).expect("newest generation");
|
||||
@@ -4048,8 +3837,10 @@ mod tests {
|
||||
let (_root, store, original) = native_expanded_backlog_store(1, false).await;
|
||||
let now = unix_now().saturating_add(60);
|
||||
let wedge = native_interleaved_expansion_wedge(&store, &original, now).await;
|
||||
let ids = native_replica_ids(&store);
|
||||
assert_eq!(wedge.last, native_replica_id(&store, 2, 1));
|
||||
let ids = (0..3)
|
||||
.flat_map(|pool_index| (0..2).map(move |set_index| replica_id(pool_index, set_index)))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(wedge.last, replica_id(2, 1));
|
||||
|
||||
// A different stable ledger at the newest key is a fork.
|
||||
let fork = claim_scanner_pause_backlog_writer(&original, now + 7).expect("forked writer generation");
|
||||
@@ -4060,7 +3851,7 @@ mod tests {
|
||||
assert_ne!(fork, wedge.acknowledged);
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
native_replica_id(&store, 1, 0),
|
||||
replica_id(1, 0),
|
||||
ScannerPauseBacklogReplicaRecord::new(
|
||||
Some(fork.clone()),
|
||||
Some(ScannerPauseBacklogCommitRecord::new(fork, ids.clone())),
|
||||
@@ -4077,7 +3868,7 @@ mod tests {
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(native_replica_snapshot(&store).await, forked, "a refused claim keeps every record");
|
||||
replace_native_pause_backlog_replica(&store, native_replica_id(&store, 1, 0), wedge.rollback.clone()).await;
|
||||
replace_native_pause_backlog_replica(&store, replica_id(1, 0), wedge.rollback.clone()).await;
|
||||
|
||||
// A missing member that a stored commit names may have lost a
|
||||
// newer stable ledger.
|
||||
@@ -4112,7 +3903,7 @@ mod tests {
|
||||
.expect_err("retirement must refuse an unproven authority");
|
||||
assert!(refused.to_string().contains("unsafe authority conflict"), "{refused}");
|
||||
assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source_bytes);
|
||||
replace_native_pause_backlog_replica(&store, native_replica_id(&store, 1, 1), wedge.rollback.clone()).await;
|
||||
replace_native_pause_backlog_replica(&store, replica_id(1, 1), wedge.rollback.clone()).await;
|
||||
|
||||
// The same records without the conflict converge and retire.
|
||||
for set in 0..2 {
|
||||
@@ -4229,7 +4020,7 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.as_ref()
|
||||
.expect("seed retains the old cohort proof");
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 0, 0), native_replica_id(&store, 0, 1)]);
|
||||
assert_eq!(committed.replicas, vec![replica_id(0, 0), replica_id(0, 1)]);
|
||||
assert_eq!(loaded.replica_count, 6);
|
||||
if loaded.healthy_replicas == loaded.replica_count {
|
||||
break loaded;
|
||||
@@ -4246,13 +4037,13 @@ mod tests {
|
||||
.authoritative_commit
|
||||
.as_ref()
|
||||
.expect("seed retains the old cohort proof");
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 0, 0), native_replica_id(&store, 0, 1)]);
|
||||
assert_eq!(committed.replicas, vec![replica_id(0, 0), replica_id(0, 1)]);
|
||||
assert_eq!(loaded.replica_count, 6);
|
||||
assert_eq!(loaded.healthy_replicas, 6);
|
||||
let seeded = loaded
|
||||
.replicas
|
||||
.iter()
|
||||
.find(|replica| replica.id == native_replica_id(&store, 2, 0))
|
||||
.find(|replica| replica.id == replica_id(2, 0))
|
||||
.expect("the canceled caller's admitted seed replica");
|
||||
assert!(matches!(
|
||||
&seeded.state,
|
||||
@@ -4264,7 +4055,7 @@ mod tests {
|
||||
let remaining = loaded
|
||||
.replicas
|
||||
.iter()
|
||||
.find(|replica| replica.id == native_replica_id(&store, 2, 1))
|
||||
.find(|replica| replica.id == replica_id(2, 1))
|
||||
.expect("the remaining seed replica");
|
||||
assert!(matches!(
|
||||
&remaining.state,
|
||||
@@ -4477,12 +4268,7 @@ mod tests {
|
||||
.prepare_scanner_pause_backlog_retirement_for_test(0, source_before.len() * 2)
|
||||
.await
|
||||
.expect("retire the old cohort's first pool");
|
||||
let surviving = [
|
||||
native_replica_id(&store, 1, 0),
|
||||
native_replica_id(&store, 1, 1),
|
||||
native_replica_id(&store, 2, 0),
|
||||
native_replica_id(&store, 2, 1),
|
||||
];
|
||||
let surviving = [replica_id(1, 0), replica_id(1, 1), replica_id(2, 0), replica_id(2, 1)];
|
||||
let newer = claim_scanner_pause_backlog_writer(&old.ledger, now + 2).expect("surviving writer");
|
||||
replace_native_pause_backlog_replica(
|
||||
&store,
|
||||
@@ -4547,7 +4333,7 @@ mod tests {
|
||||
assert_eq!(loaded.ledger, original);
|
||||
assert!(loaded.durable && loaded.stable_matches_ledger);
|
||||
let committed = loaded.authoritative_commit.expect("survivor commit proof");
|
||||
assert_eq!(committed.replicas, vec![native_replica_id(&store, 2, 0), native_replica_id(&store, 2, 1)]);
|
||||
assert_eq!(committed.replicas, vec![replica_id(2, 0), replica_id(2, 1)]);
|
||||
}
|
||||
shutdown_native_retirement_store(store).await;
|
||||
});
|
||||
@@ -5125,135 +4911,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn replica_id(pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId {
|
||||
pool_index,
|
||||
set_index,
|
||||
set_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn native_replica_id(store: &ECStore, pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId::from_set(&store.pools[pool_index].disk_set[set_index])
|
||||
.expect("native replica has an immutable format identity")
|
||||
}
|
||||
|
||||
fn native_replica_ids(store: &ECStore) -> Vec<ScannerPauseBacklogReplicaId> {
|
||||
store
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| pool.disk_set.iter())
|
||||
.map(|set| ScannerPauseBacklogReplicaId::from_set(set).expect("native replica format identity"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bound_replica_id(pool_index: usize, identity: u128) -> ScannerPauseBacklogReplicaId {
|
||||
ScannerPauseBacklogReplicaId {
|
||||
pool_index,
|
||||
set_index: 0,
|
||||
set_id: Some(uuid::Uuid::from_u128(identity)),
|
||||
}
|
||||
}
|
||||
|
||||
fn missing_replica(id: ScannerPauseBacklogReplicaId) -> ScannerPauseBacklogReplica {
|
||||
ScannerPauseBacklogReplica {
|
||||
id,
|
||||
revision: Some(DataUsageCacheRevision::Missing),
|
||||
state: ScannerPauseBacklogReplicaState::Missing,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_survives_pool_reindexing() {
|
||||
let ledger = durable_ledger(50);
|
||||
let previous = bound_replica_id(2, 22);
|
||||
let current = bound_replica_id(1, 22);
|
||||
let added = bound_replica_id(2, 33);
|
||||
let record = replica_record_for_members(&ledger, &ledger, &[previous]);
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
let loaded = select_scanner_pause_backlog_replicas(vec![
|
||||
missing_replica(bound_replica_id(0, 11)),
|
||||
decoded_replica(current, &record),
|
||||
missing_replica(added),
|
||||
])
|
||||
.expect("a surviving set keeps its commit proof after its routing index changes");
|
||||
assert_eq!(loaded.ledger, ledger);
|
||||
assert!(loaded.durable);
|
||||
assert_eq!(loaded.authoritative_commit.expect("surviving native proof").replicas, vec![current]);
|
||||
|
||||
let mut replicas = vec![retirement_replica(current, &record)];
|
||||
for id in [bound_replica_id(0, 11), added] {
|
||||
replicas.push(ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
let plan = plan_scanner_pause_backlog_retirement(1, &replicas)
|
||||
.expect("reindexed native authority permits safe retirement")
|
||||
.expect("new survivors need the existing ledger");
|
||||
assert_eq!(planned_stable_ledger(&plan), ledger);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_does_not_rebind_a_missing_set_to_its_reused_index() {
|
||||
let ledger = durable_ledger(50);
|
||||
let missing = bound_replica_id(2, 22);
|
||||
let replacement = bound_replica_id(2, 33);
|
||||
let record = replica_record_for_members(&ledger, &ledger, &[missing]);
|
||||
assert!(
|
||||
select_scanner_pause_backlog_replicas(vec![
|
||||
missing_replica(bound_replica_id(0, 11)),
|
||||
decoded_replica(replacement, &record),
|
||||
])
|
||||
.is_err(),
|
||||
"copied bytes on a new set cannot impersonate the missing commit member"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_membership_binds_only_after_its_existing_proof_is_verified() {
|
||||
let ledger = durable_ledger(50);
|
||||
let legacy = replica_record_for_members(&ledger, &ledger, &[replica_id(2, 0)]);
|
||||
assert_eq!(legacy.replica_schema_version, 1);
|
||||
let current = bound_replica_id(2, 22);
|
||||
let loaded = select_scanner_pause_backlog_replicas(vec![decoded_replica(current, &legacy)])
|
||||
.expect("unchanged legacy membership remains readable");
|
||||
let upgraded = stable_scanner_pause_backlog_record(
|
||||
&loaded.ledger,
|
||||
loaded.authoritative_commit.as_ref(),
|
||||
&scanner_pause_backlog_replica_ids(&loaded.replicas),
|
||||
);
|
||||
upgraded
|
||||
.validate()
|
||||
.expect("the replacement record binds every committed member");
|
||||
assert_eq!(upgraded.replica_schema_version, 1);
|
||||
assert_eq!(upgraded.committed.expect("bound membership").replicas, vec![current]);
|
||||
|
||||
assert!(
|
||||
select_scanner_pause_backlog_replicas(vec![
|
||||
decoded_replica(bound_replica_id(1, 22), &legacy),
|
||||
missing_replica(bound_replica_id(2, 33)),
|
||||
])
|
||||
.is_err(),
|
||||
"an already stranded legacy proof must not guess its old member's identity"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_membership_rejects_nil_duplicate_and_mixed_identities() {
|
||||
let ledger = durable_ledger(50);
|
||||
for members in [
|
||||
vec![bound_replica_id(0, 0)],
|
||||
vec![bound_replica_id(0, 11), bound_replica_id(1, 11)],
|
||||
vec![bound_replica_id(0, 11), bound_replica_id(0, 22)],
|
||||
vec![bound_replica_id(0, 11), replica_id(1, 0)],
|
||||
] {
|
||||
let record = replica_record_for_members(&ledger, &ledger, &members);
|
||||
assert_eq!(record.replica_schema_version, 1);
|
||||
assert!(record.validate().is_err());
|
||||
}
|
||||
ScannerPauseBacklogReplicaId { pool_index, set_index }
|
||||
}
|
||||
|
||||
fn decoded_replica(
|
||||
@@ -5497,7 +5155,6 @@ mod tests {
|
||||
.map(|set_index| ScannerPauseBacklogReplicaId {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
set_id: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
replica_record_for_members(stable, committed, &replicas)
|
||||
@@ -5510,7 +5167,6 @@ mod tests {
|
||||
ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: Some(serde_json::to_vec(record).expect("native record bytes")),
|
||||
}
|
||||
@@ -5638,7 +5294,6 @@ mod tests {
|
||||
replicas.extend(targets.iter().map(|id| ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: id.pool_index,
|
||||
set_index: id.set_index,
|
||||
set_id: id.set_id,
|
||||
writable: true,
|
||||
data: None,
|
||||
}));
|
||||
|
||||
@@ -113,37 +113,17 @@ pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_prein
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
setup_scanner_cycle_store_with_pool_order(
|
||||
root,
|
||||
seed_usage_baseline,
|
||||
&(0..pool_count).collect::<Vec<_>>(),
|
||||
sets_per_pool,
|
||||
drives_per_set,
|
||||
preinitialize_disks,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn setup_scanner_cycle_store_with_pool_order(
|
||||
root: &Path,
|
||||
seed_usage_baseline: bool,
|
||||
pool_order: &[usize],
|
||||
sets_per_pool: usize,
|
||||
drives_per_set: usize,
|
||||
preinitialize_disks: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let pool_count = pool_order.len();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for (pool_index, physical_pool) in pool_order.iter().enumerate() {
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..sets_per_pool {
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = if sets_per_pool == 1 {
|
||||
root.join(format!("pool{physical_pool}/disk{disk_index}"))
|
||||
root.join(format!("pool{pool_index}/disk{disk_index}"))
|
||||
} else {
|
||||
root.join(format!("pool{physical_pool}/set{set_index}/disk{disk_index}"))
|
||||
root.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}"))
|
||||
};
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
@@ -164,7 +144,7 @@ pub(super) async fn setup_scanner_cycle_store_with_pool_order(
|
||||
cmd_line: if pool_count == 1 && sets_per_pool == 1 {
|
||||
"scanner-cycle-metrics".to_string()
|
||||
} else {
|
||||
format!("scanner-cycle-metrics-pool-{physical_pool}")
|
||||
format!("scanner-cycle-metrics-pool-{pool_index}")
|
||||
},
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
});
|
||||
|
||||
@@ -72,8 +72,6 @@ Ordinary publication rechecks the selected pool against the durable pool metadat
|
||||
|
||||
Running, queued, failed, canceled, and completed decommission states exclude the source from new ordinary publication, including new multipart uploads. Previously created multipart uploads retain their drain path while the source remains non-terminal; terminal source states reject further multipart publication. Failed and canceled entries become writable for new ordinary publication only after an allowed clear operation removes that state. This check does not change repair admission or the separate fence for operations that only release capacity.
|
||||
|
||||
Bucket creation and deletion remove stale bucket usage through namespace maintenance, independently of the scanner publication pause. Failed or canceled decommission state alone does not reject that cleanup. The update checks the logical snapshot revision while holding the fixed and physical object locks, then conditionally publishes to a writable pool; a suspended source remains available for decommission to reconcile against the newer target. Cleanup preserves the scanner epoch and cycle, and retains the bucket namespace and movement guards until its physical writes finish even if the request is canceled. Namespace lease loss still fences the write. Scanner publication continues to require its existing movement admission.
|
||||
|
||||
For mixed batch deletes, only the pools selected to receive new delete markers are publication targets. Exact-version deletions on other pools remain protected by the same pool metadata read fence, without treating the retiring source or an unrelated reserved target as a destination for those markers.
|
||||
|
||||
### Shared Capacity On Healthy Targets
|
||||
@@ -145,10 +143,6 @@ Existing capacity-owner and mutation checks reconcile the pending intent before
|
||||
source cleanup; the replica exception does not clear an unknown intent, rewrite
|
||||
the native target, or change the scanner's committed-membership selection.
|
||||
|
||||
Backlog membership also carries an optional `set_id`, the first disk-slot UUID from the erasure set's persisted format. This identity survives pool removal, reordering, and disk replacement; `pool_index` and `set_index` remain routing locations. Readers resolve bound members by that UUID before checking the existing commit and rollback proofs. A new pool at a removed member's old index cannot impersonate that member. New publications bind every member, while positional records remain readable and gain identities only after their existing authority is verified. Replica schema 1 is retained; 1.0.4 readers ignore the additive identity field and still use positional membership.
|
||||
|
||||
Finish upgrading all scanner-capable nodes before changing pool membership. An older writer can replace a bound record with a positional one. If a legacy record was already stranded by pool reindexing, the missing identity cannot be guessed safely: restore the known topology in which its proof was readable, then let the updated writer bind the membership before changing it again. Do not delete or reset the backlog ledger to bypass this check. Partial writes, canceled callers, missing members, and conflicting authority remain subject to the native CAS and recovery rules above.
|
||||
|
||||
## MinIO Divergence Decisions
|
||||
|
||||
Behavior that is close to MinIO but not byte-for-byte identical. Changing either decision requires an operator compatibility note and updated characterization tests.
|
||||
|
||||
Reference in New Issue
Block a user