Merge branch 'main' into houseme/test/heal-start-deadline-contract

This commit is contained in:
houseme
2026-09-06 11:09:22 +08:00
committed by GitHub
123 changed files with 17972 additions and 937 deletions
+5
View File
@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
cargo nextest run --profile e2e-distributed -p e2e_test
# Replication nightly lane; awscurl is required for STS paths
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2024 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::path::Path;
use std::process::Command;
fn git(root: &Path, args: &[&str]) -> Option<String> {
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
output
.status
.success()
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
}
fn emit(name: &str, value: &str) {
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
}
fn main() {
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
let root = Path::new(&manifest).join("../..");
// Cover dependency/common sources as well as this crate. HEAD/ref/index
// changes must refresh identity even when no Rust source mtime changes.
for path in [
"crates",
"rustfs",
"Cargo.toml",
"Cargo.lock",
"rust-toolchain.toml",
".cargo",
".config",
] {
println!("cargo:rerun-if-changed={}", root.join(path).display());
}
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
git_paths.push(reference);
}
for path in git_paths {
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
let path = Path::new(&path);
let path = if path.is_absolute() {
path.to_owned()
} else {
root.join(path)
};
if path.exists() {
println!("cargo:rerun-if-changed={}", path.display());
}
}
}
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
let mut features = std::env::vars()
.filter_map(|(key, _)| {
key.strip_prefix("CARGO_FEATURE_")
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
})
.collect::<Vec<_>>();
features.sort();
emit("COMMIT", &revision);
emit("DIRTY", if dirty { "true" } else { "false" });
emit("LOCK", &lock);
emit("FEATURES", &features.join(","));
for name in ["TARGET", "PROFILE"] {
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
}
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
emit("RUSTFLAGS_HEX", &flags);
}
+13 -3
View File
@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
/// tests need this lower-level record to prove that the rebuilt target holds the
/// `xl.meta` selected for a specific version and every `part.N` it declares.
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
pub(crate) struct VersionShardCensus {
pub version_id: Option<String>,
pub has_xl_meta: bool,
pub data_dir: Option<String>,
pub erasure_index: Option<usize>,
pub data_blocks: Option<usize>,
pub parity_blocks: Option<usize>,
pub expected_part_numbers: BTreeSet<usize>,
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
pub inline_data_fingerprint: Option<PartShardFingerprint>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
pub(crate) struct PartShardFingerprint {
pub size: u64,
pub sha256: String,
@@ -88,13 +90,15 @@ impl VersionShardCensus {
&& manifest.is_complete()
&& self.data_dir == manifest.data_dir
&& self.erasure_index == manifest.erasure_index
&& self.data_blocks == manifest.data_blocks
&& self.parity_blocks == manifest.parity_blocks
&& self.expected_part_numbers == manifest.expected_part_numbers
&& self.present_part_fingerprints == manifest.present_part_fingerprints
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
}
}
fn sha256_hex(data: &[u8]) -> String {
pub(crate) fn sha256_hex(data: &[u8]) -> String {
let digest = Sha256::digest(data);
digest.iter().map(|byte| format!("{byte:02x}")).collect()
}
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
has_xl_meta: false,
data_dir: None,
erasure_index: None,
data_blocks: None,
parity_blocks: None,
expected_part_numbers: BTreeSet::new(),
present_part_fingerprints: BTreeMap::new(),
inline_data_fingerprint: None,
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
has_xl_meta: true,
data_dir,
erasure_index,
data_blocks: Some(file_info.erasure.data_blocks),
parity_blocks: Some(file_info.erasure.parity_blocks),
expected_part_numbers,
present_part_fingerprints,
inline_data_fingerprint,
@@ -413,6 +421,8 @@ mod tests {
has_xl_meta: true,
data_dir: Some("data-dir".to_string()),
erasure_index: Some(3),
data_blocks: Some(2),
parity_blocks: Some(2),
expected_part_numbers: BTreeSet::from([1]),
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
inline_data_fingerprint: None,
+63
View File
@@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment {
Ok(())
}
/// Append a new single-node erasure pool to a stopped multi-pool cluster.
///
/// Used to simulate pool expansion on localhost: every pool already owns
/// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
/// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
/// allocated a fresh port and empty drive directories; callers must
/// [`Self::start`] afterwards so every process picks up the extended
/// volumes argument. Existing data directories are left untouched.
pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
if self.nodes.iter().any(|node| node.process.is_some()) {
return Err("stop the cluster before appending a pool".into());
}
if self.topology.drives_per_node < 2 {
return Err(
"append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
.into(),
);
}
let mut pools = self.topology.normalized_pools();
for (pool_idx, nodes) in pools.iter().enumerate() {
if nodes.len() != 1 {
return Err(format!(
"pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
nodes.len()
)
.into());
}
}
let new_idx = self.nodes.len();
let port = RustFSTestEnvironment::find_available_port().await?;
let address = format!("127.0.0.1:{port}");
let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
.map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
.collect();
for dir in &data_dirs {
fs::create_dir_all(dir).await?;
}
self.nodes.push(ClusterNode {
url: format!("http://{address}"),
address,
data_dir: data_dirs[0].clone(),
data_dirs,
pool_idx: pools.len(),
process: None,
});
pools.push(vec![new_idx]);
self.topology.node_count = self.nodes.len();
self.topology.pools = pools;
self.node_extra_env.push(Vec::new());
self.node_capture_log_paths.push(None);
self.volume_proxy_addresses.push(None);
if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
self.extra_env
.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
}
Ok(new_idx)
}
/// Gracefully stop one cluster node and wait for its process to exit.
///
/// This is intentionally separate from [`Self::stop_node`]: the latter is
+6 -2
View File
@@ -35,11 +35,15 @@ where
{
let mut last_usage = DataUsageInfo::default();
let mut last_query_error = None;
for _ in 0..45 {
for _ in 0..90 {
match get_data_usage_info(env).await {
Ok(usage) => {
last_query_error = None;
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
if usage.is_complete_bucket_usage_snapshot()
&& usage.usage_snapshot_converged != Some(false)
&& usage.buckets_usage.contains_key(bucket)
&& predicate(&usage)
{
return Ok(usage);
}
last_usage = usage;
@@ -0,0 +1,222 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
wait_for_ready, wait_until,
};
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
use crate::common::{build_test_s3_config, init_logging};
use crate::fault_proxy::FaultMode;
use aws_sdk_s3::Client;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{Barrier, mpsc};
use tokio::time::timeout;
#[tokio::test]
async fn kill_and_restart_node_preserves_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("killnode");
dist.create_bucket(&bucket).await?;
let body = vec![0x11u8; 128 * 1024];
put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
dist.cluster.stop_node(3)?;
retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
dist.cluster.start_node(3).await?;
wait_for_ready(&dist.cluster).await?;
assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
Ok(())
}
#[tokio::test]
async fn full_cluster_restart_preserves_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("pwr");
dist.create_bucket(&bucket).await?;
let body = vec![0x44u8; 64 * 1024];
put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
dist.cluster.stop();
dist.cluster.start().await?;
wait_for_ready(&dist.cluster).await?;
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
}
Ok(())
}
#[tokio::test]
async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
init_logging();
let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
let bucket = unique_bucket("baddrive");
dist.create_bucket(&bucket).await?;
let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
assert!(
baseline.is_complete(),
"replacement target did not hold a complete baseline shard: {baseline:?}"
);
assert!(
!baseline.expected_part_numbers.is_empty(),
"replacement witness must use physical part shards: {baseline:?}"
);
dist.cluster.stop_node(0)?;
let format_path = replaced_drive.join(".rustfs.sys/format.json");
let format = std::fs::read(&format_path)?;
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
std::fs::rename(&replaced_drive, &retired_drive)?;
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
std::fs::write(&format_path, format)?;
let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
dist.cluster.start_node(0).await?;
wait_for_ready(&dist.cluster).await?;
let heal_body =
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
wait_until(
Duration::from_secs(90),
|| async {
let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
Ok(healed.matches_manifest(&baseline))
},
"fresh replacement contains the original complete shard manifest",
)
.await?;
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
}
Ok(())
}
#[tokio::test]
async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("getkill");
dist.create_bucket(&bucket).await?;
let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
let worker_count = 12;
let release = Arc::new(Barrier::new(worker_count + 1));
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let mut handles = Vec::new();
for idx in 0..worker_count {
let client = live[idx % live.len()].clone();
let bucket = bucket.clone();
let body = body.clone();
let release = release.clone();
let started_tx = started_tx.clone();
handles.push(tokio::spawn(async move {
let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
if response.content_length() != Some(body.len() as i64) {
return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
format!("worker {idx} received a wrong content length").into(),
);
}
started_tx.send(idx)?;
release.wait().await;
let actual = response.body.collect().await?.into_bytes();
if actual.as_ref() != body.as_slice() {
return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
}
Ok(())
}));
}
drop(started_tx);
for _ in 0..worker_count {
timeout(Duration::from_secs(30), started_rx.recv())
.await?
.ok_or("a streaming GET exited before reaching the kill barrier")?;
}
dist.cluster.stop_node(3)?;
release.wait().await;
for handle in handles {
handle.await??;
}
dist.cluster.start_node(3).await?;
wait_for_ready(&dist.cluster).await?;
assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
Ok(())
}
#[tokio::test]
async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
let proxied_url = format!("http://{}", proxy.local_addr());
let proxied_client = Client::from_conf(build_test_s3_config(
&proxied_url,
&dist.cluster.access_key,
&dist.cluster.secret_key,
None,
"distributed-network-chaos",
));
let result: TestResult = async {
let bucket = unique_bucket("netfault");
dist.create_bucket(&bucket).await?;
let baseline = payload_for("network/baseline.bin", 1024 * 1024);
put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
proxy.set_mode(FaultMode::Blackhole);
assert_eq!(proxy.mode(), FaultMode::Blackhole);
if let Ok(Ok(_)) = timeout(
Duration::from_secs(5),
proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
)
.await
{
return Err("blackholed node endpoint unexpectedly completed a GET".into());
}
let during = payload_for("network/during.bin", 1024 * 1024);
timeout(Duration::from_secs(30), async {
put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
})
.await??;
proxy.set_mode(FaultMode::Pass);
retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
Ok(())
}
.await;
proxy.set_mode(FaultMode::Pass);
proxy.shutdown().await;
result
}
@@ -0,0 +1,98 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use std::collections::BTreeSet;
use std::sync::Arc;
use tokio::sync::Barrier;
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
let mut handles = Vec::new();
for idx in 0..worker_count {
let clients = clients.clone();
let barrier = barrier.clone();
let bucket = bucket.clone();
handles.push(tokio::spawn(async move {
barrier.wait().await;
let writer = &clients[idx % clients.len()];
let reader = &clients[(idx + 1) % clients.len()];
let copier = &clients[(idx + 2) % clients.len()];
let mut retained = Vec::with_capacity(rounds);
for round in 0..rounds {
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
put_object(writer, &bucket, &key, body.clone()).await?;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_object_bytes(reader, &bucket, &key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
retained.push((copy_key, body));
}
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
}));
}
let mut inventory = Vec::new();
for handle in handles {
inventory.extend(handle.await??);
}
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
}
Ok(())
}
@@ -0,0 +1,74 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
};
use crate::common::init_logging;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
#[tokio::test]
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("concdecom");
dist.create_bucket(&bucket).await?;
let baseline_client = dist.client(0)?;
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
let clients = Arc::new(dist.clients()?);
let barrier = Arc::new(Barrier::new(17));
let mut handles = Vec::new();
for idx in 0..16 {
let clients = clients.clone();
let barrier = barrier.clone();
let bucket = bucket.clone();
handles.push(tokio::spawn(async move {
barrier.wait().await;
let client = &clients[idx % clients.len()];
let key = format!("live/{idx:02}.bin");
let body = payload_for(&key, 8 * 1024);
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
}));
}
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
barrier.wait().await;
let mut live_objects = Vec::new();
for handle in handles {
live_objects.push(handle.await??);
}
let status = decommission_status_json(&dist.cluster).await?;
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
}
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let checker = dist.client(2)?;
assert_inventory(&checker, &bucket, &inventory).await?;
for (key, body) in live_objects {
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
}
Ok(())
}
@@ -0,0 +1,156 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
};
use crate::common::init_logging;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::time::Duration;
#[tokio::test]
async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("integrity");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
enable_versioning(&client, &bucket).await?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
let versioned_key = "history/versioned.bin";
let version_one = b"historical bytes before data movement".to_vec();
let version_two = b"current bytes before data movement".to_vec();
let version_one_id = client
.put_object()
.bucket(&bucket)
.key(versioned_key)
.body(ByteStream::from(version_one.clone()))
.send()
.await?
.version_id()
.ok_or("historical PUT omitted version ID")?
.to_string();
let version_two_id = client
.put_object()
.bucket(&bucket)
.key(versioned_key)
.body(ByteStream::from(version_two.clone()))
.send()
.await?
.version_id()
.ok_or("current PUT omitted version ID")?
.to_string();
let multipart_key = "multipart/moved.bin";
let first_part = vec![0x31; 5 * 1024 * 1024];
let second_part = vec![0x72; 1024 * 1024];
let upload = client
.create_multipart_upload()
.bucket(&bucket)
.key(multipart_key)
.send()
.await?;
let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
let uploaded_one = client
.upload_part()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(first_part.clone()))
.send()
.await?;
let uploaded_two = client
.upload_part()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.part_number(2)
.body(ByteStream::from(second_part.clone()))
.send()
.await?;
client
.complete_multipart_upload()
.bucket(&bucket)
.key(multipart_key)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
.build(),
)
.build(),
)
.send()
.await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let after_client = dist.client(2)?;
assert_inventory(&after_client, &bucket, &inventory).await?;
for (key, expected_hash) in before {
let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
let body = got.body.collect().await?.into_bytes();
assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
}
for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
let got = after_client
.get_object()
.bucket(&bucket)
.key(versioned_key)
.version_id(version_id)
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
}
let mut expected_multipart = first_part;
expected_multipart.extend_from_slice(&second_part);
let got_multipart = after_client
.get_object()
.bucket(&bucket)
.key(multipart_key)
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(
sha256_hex(got_multipart.as_ref()),
sha256_hex(&expected_multipart),
"multipart checksum changed after decommission"
);
Ok(())
}
@@ -0,0 +1,81 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
};
use crate::common::init_logging;
use std::time::Duration;
#[tokio::test]
async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("expand");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
assert_inventory(&client, &bucket, &inventory).await?;
for expected_nodes in 2..=4 {
let new_node = dist.append_pool_and_restart().await?;
assert_eq!(new_node + 1, expected_nodes);
assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
}
assert_eq!(dist.cluster.nodes.len(), 4);
// Prove that the expanded pool map is durable, and clear any recovery
// latch raised while the newly-added pool replicas converged.
dist.restart_current_binary_gracefully().await?;
let after_expand = dist.client(0)?;
assert_inventory(&after_expand, &bucket, &inventory).await?;
let peer = dist.client(3)?;
assert_inventory(&peer, &bucket, &inventory).await?;
let rebalance_id = start_rebalance(&dist.cluster).await?;
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
assert_inventory(&peer, &bucket, &inventory).await?;
Ok(())
}
#[tokio::test]
async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("decom");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
let pools_before = list_pools_json(&dist.cluster).await?;
let pool_count = pools_before
.as_array()
.map(Vec::len)
.or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
.ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
let after = dist.client(2)?;
assert_inventory(&after, &bucket, &inventory).await?;
Ok(())
}
@@ -0,0 +1,149 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
};
use crate::common::init_logging;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use std::time::Duration;
#[tokio::test]
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("extra");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let key = "multipart.bin";
let part1 = vec![0x41u8; 5 * 1024 * 1024];
let part2 = vec![0x42u8; 5 * 1024 * 1024];
let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
let uploaded1 = client
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from(part1.clone()))
.send()
.await?;
let uploaded2 = client
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.part_number(2)
.body(ByteStream::from(part2.clone()))
.send()
.await?;
client
.complete_multipart_upload()
.bucket(&bucket)
.key(key)
.upload_id(&upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(uploaded1.e_tag().unwrap_or_default())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(uploaded2.e_tag().unwrap_or_default())
.build(),
)
.build(),
)
.send()
.await?;
let mut expected = part1;
expected.extend_from_slice(&part2);
for node_idx in 0..dist.cluster.nodes.len() {
assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
}
put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
let mut seen = Vec::new();
for node_idx in 0..dist.cluster.nodes.len() {
let listed = dist
.client(node_idx)?
.list_objects_v2()
.bucket(&bucket)
.prefix("list/")
.send()
.await?;
let keys: Vec<String> = listed
.contents()
.iter()
.filter_map(|object| object.key().map(str::to_string))
.collect();
seen.push(keys);
}
for keys in &seen[1..] {
assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
}
let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
assert_eq!(got, b"a");
Ok(())
}
#[tokio::test]
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("listed");
dist.create_bucket(&bucket).await?;
put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
for node_idx in 0..dist.cluster.nodes.len() {
let client = dist.client(node_idx)?;
let name = bucket.clone();
wait_until(
Duration::from_secs(20),
|| {
let client = client.clone();
let name = name.clone();
async move {
let listed = client.list_buckets().send().await?;
Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
}
},
&format!("node {node_idx} lists {bucket}"),
)
.await?;
wait_until(
Duration::from_secs(20),
|| {
let client = dist.client(node_idx).expect("client");
let name = bucket.clone();
async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
},
&format!("node {node_idx} reads seed.bin"),
)
.await?;
}
Ok(())
}
File diff suppressed because it is too large Load Diff
+35
View File
@@ -0,0 +1,35 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! 4-node 4-drive distributed e2e coverage.
//!
//! Selected by `[profile.e2e-distributed]` and run from
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
//! each case starts four real `rustfs` processes.
mod chaos_test;
mod concurrency_stability_test;
mod concurrent_data_movement_test;
mod data_integrity_movement_test;
mod expand_decommission_rebalance_test;
mod extra_test;
mod harness;
mod object_lock_test;
mod observability_test;
mod replication_quota_test;
mod s3_basic_test;
mod s3_during_data_movement_test;
mod site_replication_test;
mod upgrade_test;
mod versioning_test;
@@ -0,0 +1,219 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
use crate::common::init_logging;
use crate::object_lock::common::{
delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::delete_object::DeleteObjectError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
ObjectLockRule,
};
use chrono::{Duration as ChronoDuration, Utc};
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if code == Some("AccessDenied") {
Ok(())
} else {
Err(format!("{context}: expected AccessDenied, got {error:?}").into())
}
}
async fn expect_versioned_delete_denied(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
bypass: bool,
context: &str,
) -> TestResult {
match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
Err(error) => delete_denied(error.as_ref(), context),
}
}
#[tokio::test]
async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let client = dist.client(0)?;
let peer = dist.client(2)?;
let bucket = unique_bucket("objlock");
client
.create_bucket()
.bucket(&bucket)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
let retain_until = Utc::now() + ChronoDuration::days(1);
let compliance_key = "compliance.bin";
let compliance_version = put_object_with_retention(
&client,
&bucket,
compliance_key,
b"locked-compliance",
ObjectLockRetentionMode::Compliance,
retain_until,
)
.await?;
// Unversioned DELETE is allowed: it only creates a delete marker. WORM
// applies to a specific version id.
let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
assert_eq!(
marker.delete_marker(),
Some(true),
"unversioned DELETE on a locked object must create a delete marker"
);
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
.await?;
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
let governance_key = "governance.bin";
let governance_version = put_object_with_retention(
&client,
&bucket,
governance_key,
b"locked-governance",
ObjectLockRetentionMode::Governance,
retain_until,
)
.await?;
expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
.await?;
delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
let deleted_governance = peer
.head_object()
.bucket(&bucket)
.key(governance_key)
.version_id(&governance_version)
.send()
.await
.expect_err("GOVERNANCE bypass must remove the retained version");
assert_eq!(
deleted_governance.raw_response().map(|response| response.status().as_u16()),
Some(404),
"deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
);
let hold_key = "legal-hold.bin";
let hold_version =
put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
Ok(())
}
#[tokio::test]
async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let bucket = unique_bucket("default-lock");
writer
.create_bucket()
.bucket(&bucket)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
let key = "default-governance.bin";
let put = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from_static(b"default retention payload"))
.send()
.await?;
let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
let default_retention = config
.object_lock_configuration()
.and_then(|configuration| configuration.rule())
.and_then(|rule| rule.default_retention())
.ok_or("GetObjectLockConfiguration omitted default retention")?;
assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
assert_eq!(default_retention.days(), Some(1));
let retention = reader
.get_object_retention()
.bucket(&bucket)
.key(key)
.version_id(version_id)
.send()
.await?;
let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
let retain_until = retention
.retain_until_date()
.ok_or("default retention omitted retain-until date")?;
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
.await?;
let plain_bucket = unique_bucket("no-lock");
dist.create_bucket(&plain_bucket).await?;
let configuration = ObjectLockConfiguration::builder()
.object_lock_enabled(ObjectLockEnabled::Enabled)
.rule(
ObjectLockRule::builder()
.default_retention(
DefaultRetention::builder()
.mode(ObjectLockRetentionMode::Governance)
.days(1)
.build(),
)
.build(),
)
.build();
let error = writer
.put_object_lock_configuration()
.bucket(&plain_bucket)
.object_lock_configuration(configuration)
.send()
.await
.expect_err("an unversioned bucket must reject Object Lock enablement");
let service_error = error
.as_service_error()
.ok_or("non-lock bucket rejection was not an S3 service error")?;
assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
assert_eq!(
service_error.message(),
Some("Object Lock configuration cannot be enabled on existing buckets"),
"unexpected error: {error:?}"
);
Ok(())
}
@@ -0,0 +1,236 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
use crate::common::{admin_request, init_logging, local_http_client};
use aws_sdk_s3::operation::RequestId;
use aws_sdk_s3::primitives::ByteStream;
use bytes::Bytes;
use http::Method;
use http_body_util::{BodyExt, Empty};
use hyper::body::Incoming;
use hyper::service::service_fn;
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use local_ip_address::local_ip;
use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use serde_json::Value;
use std::convert::Infallible;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio::time::{Instant, timeout};
async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
let listener = TcpListener::bind("0.0.0.0:0").await?;
let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
let (tx, rx) = mpsc::unbounded_channel();
let handle = tokio::spawn(async move {
loop {
let Ok((stream, _)) = listener.accept().await else {
return;
};
let tx = tx.clone();
tokio::spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
let tx = tx.clone();
async move {
let method = request.method().clone();
if let Ok(body) = request.into_body().collect().await
&& method == Method::POST
&& let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
{
if let Some(records) = payload["Records"].as_array() {
for entry in records {
let _ = tx.send(entry.clone());
}
} else {
let _ = tx.send(payload);
}
}
Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
}
});
let _ = hyper::server::conn::http1::Builder::new()
.serve_connection(TokioIo::new(stream), service)
.await;
});
}
});
Ok((endpoint, rx, handle))
}
async fn wait_for_audit_entry(
rx: &mut mpsc::UnboundedReceiver<Value>,
bucket: &str,
key: &str,
request_id: &str,
) -> TestResult<Value> {
let deadline = Instant::now() + Duration::from_secs(30);
let mut seen = Vec::new();
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(format!(
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
seen.len()
)
.into());
}
let entry = match timeout(remaining, rx.recv()).await {
Ok(Some(entry)) => entry,
Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
Err(_) => {
return Err(format!(
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
seen.len()
)
.into());
}
};
if entry["api"]["name"].as_str() == Some("s3:PutObject")
&& entry["api"]["bucket"].as_str() == Some(bucket)
&& entry["api"]["object"].as_str() == Some(key)
&& entry["requestID"].as_str() == Some(request_id)
{
return Ok(entry);
}
if seen.len() < 8 {
seen.push(format!(
"api={:?} bucket={:?} object={:?} requestID={:?}",
entry["api"]["name"].as_str(),
entry["api"]["bucket"].as_str(),
entry["api"]["object"].as_str(),
entry["requestID"].as_str()
));
}
}
}
#[tokio::test]
async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
init_logging();
let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
let audit_env = [
("RUSTFS_AUDIT_ENABLE", "true"),
("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
(ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
];
let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
for node_idx in 0..dist.cluster.nodes.len() {
let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
dist.cluster
.set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
}
dist.cluster.start().await?;
wait_for_ready(&dist.cluster).await?;
let http = local_http_client();
for node in &dist.cluster.nodes {
for probe in ["ready", "live"] {
let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
assert!(
response.status().is_success(),
"node {} {probe} probe failed: {}",
node.address,
response.status()
);
}
}
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
let info: Value = serde_json::from_str(&info_body)?;
let servers = info["info"]["servers"]
.as_array()
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
let storage: Value = serde_json::from_str(&storage_body)?;
let disks = storage["info"]["disks"]
.as_array()
.ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
assert!(
disks.iter().all(|disk| {
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
&& disk["runtimeState"]
.as_str()
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
}),
"storageinfo reported a drive that was not healthy and online: {storage}"
);
for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
let (status, metrics_body) = admin_request(
&node.url,
Method::GET,
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
None,
&dist.cluster.access_key,
&dist.cluster.secret_key,
)
.await?;
assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
let sample: RealtimeMetrics = serde_json::from_str(
metrics_body
.lines()
.next()
.ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
)?;
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
}
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
let targets: Value = serde_json::from_str(&targets_body)?;
let configured = targets["audit_endpoints"]
.as_array()
.ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
.iter()
.any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
assert!(configured, "configured audit webhook was missing: {targets}");
let bucket = unique_bucket("audit");
dist.create_bucket(&bucket).await?;
let key = "correlated/audit-object.bin";
let put = dist
.client(2)?
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from_static(b"distributed audit payload"))
.send()
.await?;
let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
assert_eq!(
audit["api"]["status_code"].as_i64(),
Some(200),
"audit entry did not report success: {audit}"
);
assert!(
!audit.to_string().contains(&dist.cluster.secret_key),
"audit entry leaked the root secret key"
);
collector.abort();
Ok(())
}
@@ -0,0 +1,191 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use http::Method;
use std::time::Duration;
async fn wait_for_replication_status(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
expected: &[&str],
timeout: Duration,
) -> TestResult {
wait_until(
timeout,
|| async {
let head = client.head_object().bucket(bucket).key(key).send().await?;
Ok(head
.replication_status()
.is_some_and(|status| expected.contains(&status.as_str())))
},
&format!("replication status for {bucket}/{key} in {expected:?}"),
)
.await
}
#[tokio::test]
async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
init_logging();
let (source, mut target) = DistCluster::start_replication_pair().await?;
let source_bucket = unique_bucket("replsrc");
let target_bucket = unique_bucket("repldst");
source.create_bucket(&source_bucket).await?;
target.create_bucket(&target_bucket).await?;
let source_client = source.client(0)?;
let target_client = target.client(0)?;
enable_versioning(&source_client, &source_bucket).await?;
enable_versioning(&target_client, &target_bucket).await?;
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
let key = "replicated/metadata-and-tags.bin";
let body = b"distributed-bucket-replication".to_vec();
source_client
.put_object()
.bucket(&source_bucket)
.key(key)
.metadata("origin", "four-node-source")
.tagging("suite=distributed&shape=metadata")
.body(ByteStream::from(body.clone()))
.send()
.await?;
wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
let peer_read = target.client(3)?;
wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
assert_eq!(
replica_head
.metadata()
.and_then(|metadata| metadata.get("origin"))
.map(String::as_str),
Some("four-node-source")
);
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
assert_eq!(tags.get("suite"), Some(&"distributed"));
assert_eq!(tags.get("shape"), Some(&"metadata"));
target.cluster.stop();
let outage_key = "replicated/queued-during-target-outage.bin";
let outage_body = b"retry-after-target-restart".to_vec();
put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
wait_for_replication_status(
&source_client,
&source_bucket,
outage_key,
&["PENDING", "FAILED"],
Duration::from_secs(30),
)
.await?;
target.cluster.start().await?;
wait_for_ready(&target.cluster).await?;
wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
Ok(())
}
#[tokio::test]
async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
init_logging();
let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
let bucket = unique_bucket("quota");
dist.create_bucket(&bucket).await?;
set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
let client = dist.client(1)?;
retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
wait_until(
Duration::from_secs(30),
|| async {
let (status, body) = super::harness::cluster_admin(
&dist.cluster,
Method::GET,
&format!("/rustfs/admin/v3/quota-stats/{bucket}"),
None,
)
.await?;
if !status.is_success() {
return Ok(false);
}
let stats: serde_json::Value =
serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
let usage = stats
.get("current_usage")
.and_then(serde_json::Value::as_u64)
.ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
Ok(usage >= 1024)
},
"quota stats observe small object",
)
.await?;
let oversized_key = "too-big.bin";
let error = client
.put_object()
.bucket(&bucket)
.key(oversized_key)
.body(vec![0u8; 16 * 1024].into())
.send()
.await
.expect_err("hard quota must reject the oversized PUT");
let service_error = error
.as_service_error()
.ok_or("quota rejection was not an S3 service error")?;
assert_eq!(
error.raw_response().map(|response| response.status().as_u16()),
Some(400),
"quota rejection must be HTTP 400: {error:?}"
);
assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
assert!(
service_error
.message()
.is_some_and(|message| message.starts_with("Bucket quota exceeded")),
"PUT must fail specifically at quota admission: {error:?}"
);
let missing = client
.head_object()
.bucket(&bucket)
.key(oversized_key)
.send()
.await
.expect_err("an object rejected by quota must not become visible");
assert_eq!(
missing.raw_response().map(|response| response.status().as_u16()),
Some(404),
"quota-rejected object returned an unexpected HEAD result: {missing:?}"
);
let listed = client.list_objects_v2().bucket(&bucket).send().await?;
assert!(
listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
"quota-rejected key leaked into ListObjectsV2"
);
Ok(())
}
@@ -0,0 +1,258 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
use crate::common::{init_logging, local_http_client};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::presigning::PresigningConfig;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
use std::time::Duration;
#[tokio::test]
async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("s3basic");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let key = "dir/object.bin";
let body = vec![0xA5u8; 256 * 1024];
put_object(&writer, &bucket, key, body.clone()).await?;
let head = reader.head_object().bucket(&bucket).key(key).send().await?;
assert_eq!(head.content_length(), Some(body.len() as i64));
assert_object_bytes(&reader, &bucket, key, &body).await?;
let ranged = reader
.get_object()
.bucket(&bucket)
.key(key)
.range("bytes=0-15")
.send()
.await?;
let ranged_body = ranged.body.collect().await?.into_bytes();
assert_eq!(ranged_body.as_ref(), &body[..16]);
let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(keys, vec![key]);
let copy_key = "dir/object-copy.bin";
reader
.copy_object()
.bucket(&bucket)
.key(copy_key)
.copy_source(format!("{bucket}/{key}"))
.metadata_directive(MetadataDirective::Copy)
.send()
.await?;
assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
let moved_key = "dir/object-moved.bin";
writer
.copy_object()
.bucket(&bucket)
.key(moved_key)
.copy_source(format!("{bucket}/{copy_key}"))
.send()
.await?;
writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
match writer.head_object().bucket(&bucket).key(copy_key).send().await {
Ok(_) => return Err("copied source still present after rename delete".into()),
Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
Err(error) => return Err(error.into()),
}
assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
let presigned = writer
.get_object()
.bucket(&bucket)
.key(key)
.presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
.await?;
let response = local_http_client().get(presigned.uri().to_string()).send().await?;
assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
let presigned_body = response.bytes().await?;
assert_eq!(presigned_body.as_ref(), body.as_slice());
let empty_key = "empty";
put_object(&writer, &bucket, empty_key, Vec::new()).await?;
let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
assert!(empty.is_empty());
let deleted = writer
.delete_objects()
.bucket(&bucket)
.delete(
Delete::builder()
.objects(ObjectIdentifier::builder().key(key).build()?)
.objects(ObjectIdentifier::builder().key(moved_key).build()?)
.objects(ObjectIdentifier::builder().key(empty_key).build()?)
.build()?,
)
.send()
.await?;
assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
Ok(())
}
#[tokio::test]
async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("s3matrix");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
let special_key = "unicode/测试 space+percent%25.txt";
let special_body = b"metadata and tagging survive distributed routing".to_vec();
let put = writer
.put_object()
.bucket(&bucket)
.key(special_key)
.metadata("test-meta", "distributed")
.tagging("purpose=compatibility&scope=four-by-four")
.body(ByteStream::from(special_body.clone()))
.send()
.await?;
let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
assert_eq!(
head.metadata()
.and_then(|metadata| metadata.get("test-meta"))
.map(String::as_str),
Some("distributed")
);
assert_eq!(head.e_tag(), Some(etag.as_str()));
let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
let actual_tags: std::collections::BTreeMap<_, _> = tags
.tag_set()
.iter()
.map(|tag| (tag.key().to_string(), tag.value().to_string()))
.collect();
assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
let conditional = reader
.get_object()
.bucket(&bucket)
.key(special_key)
.if_match(&etag)
.send()
.await?;
assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
let invalid_range = reader
.get_object()
.bucket(&bucket)
.key(special_key)
.range("bytes=999999-1000000")
.send()
.await
.expect_err("an unsatisfiable range must fail");
assert_eq!(
invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidRange"),
"unexpected invalid-range error: {invalid_range:?}"
);
let upload_key = "multipart/aborted.bin";
let upload = writer
.create_multipart_upload()
.bucket(&bucket)
.key(upload_key)
.send()
.await?;
let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
writer
.upload_part()
.bucket(&bucket)
.key(upload_key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
.send()
.await?;
let pending = reader
.list_multipart_uploads()
.bucket(&bucket)
.prefix("multipart/")
.send()
.await?;
assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
writer
.abort_multipart_upload()
.bucket(&bucket)
.key(upload_key)
.upload_id(upload_id)
.send()
.await?;
let after_abort = reader
.list_multipart_uploads()
.bucket(&bucket)
.prefix("multipart/")
.send()
.await?;
assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
let aborted_head = reader
.head_object()
.bucket(&bucket)
.key(upload_key)
.send()
.await
.expect_err("aborted multipart upload must not create an object");
assert_eq!(
aborted_head.raw_response().map(|response| response.status().as_u16()),
Some(404),
"aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
);
for index in 0..113 {
let key = format!("page/{index:04}.txt");
put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
}
let mut token = None;
let mut paged_keys = Vec::new();
loop {
let page = reader
.list_objects_v2()
.bucket(&bucket)
.prefix("page/")
.max_keys(37)
.set_continuation_token(token.take())
.send()
.await?;
paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
if page.is_truncated() != Some(true) {
break;
}
token = Some(
page.next_continuation_token()
.ok_or("truncated ListObjectsV2 page omitted next continuation token")?
.to_string(),
);
}
assert_eq!(paged_keys.len(), 113);
let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
Ok(())
}
@@ -0,0 +1,94 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
};
use crate::common::init_logging;
use std::time::Duration;
#[tokio::test]
async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
init_logging();
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
let bucket = unique_bucket("s3move");
dist.create_bucket(&bucket).await?;
let client = dist.client(0)?;
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
dist.expand_to_four_pools().await?;
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
let live = dist.client(2)?;
retrying_put(
&live,
&bucket,
"during-decommission.bin",
b"written-while-decommissioning".to_vec(),
Duration::from_secs(30),
)
.await?;
retrying_get_equals(
&live,
&bucket,
"during-decommission.bin",
b"written-while-decommissioning",
Duration::from_secs(30),
)
.await?;
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
assert!(
listed
.contents()
.iter()
.any(|object| object.key() == Some("during-decommission.bin")),
"list during decommission missed the newly written key"
);
let status = decommission_status_json(&dist.cluster).await?;
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
}
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
assert_inventory(&live, &bucket, &inventory).await?;
let rebalance_id = start_rebalance(&dist.cluster).await?;
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
retrying_put(
&live,
&bucket,
"during-rebalance.bin",
b"written-while-rebalancing".to_vec(),
Duration::from_secs(30),
)
.await?;
retrying_get_equals(
&live,
&bucket,
"during-rebalance.bin",
b"written-while-rebalancing",
Duration::from_secs(30),
)
.await?;
let status = rebalance_status_json(&dist.cluster).await?;
if !rebalance_running_with_progress(&status, &rebalance_id)? {
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
}
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
Ok(())
}
@@ -0,0 +1,128 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{
DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
wait_until,
};
use crate::common::{init_logging, signed_request};
use http::{Method, StatusCode};
use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
use std::time::Duration;
async fn site_replication_add(
cluster: &crate::common::RustFSTestClusterEnvironment,
sites: &[PeerSite],
) -> TestResult<ReplicateAddStatus> {
let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
let response = signed_request(
Method::PUT,
&url,
&cluster.access_key,
&cluster.secret_key,
Some(serde_json::to_vec(sites)?),
Some("application/json"),
)
.await?;
if response.status() != StatusCode::OK {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(format!("site replication add failed: {status} {body}").into());
}
Ok(serde_json::from_slice(&response.bytes().await?)?)
}
async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
Ok(serde_json::from_str(&body)?)
}
async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
wait_until(
Duration::from_secs(30),
|| async {
let info = site_replication_info(cluster).await?;
Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
},
"site replication enabled with two synchronized sites",
)
.await
}
#[tokio::test]
async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
init_logging();
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
let bucket = unique_bucket("siterepl");
site_a.create_bucket(&bucket).await?;
site_b.create_bucket(&bucket).await?;
let client_a = site_a.client(0)?;
let client_b = site_b.client(0)?;
enable_versioning(&client_a, &bucket).await?;
enable_versioning(&client_b, &bucket).await?;
let sites = vec![
PeerSite {
name: "site-a".to_string(),
endpoint: site_a.cluster.nodes[0].url.clone(),
access_key: site_a.cluster.access_key.clone(),
secret_key: site_a.cluster.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "site-b".to_string(),
endpoint: site_b.cluster.nodes[0].url.clone(),
access_key: site_b.cluster.access_key.clone(),
secret_key: site_b.cluster.secret_key.clone(),
..Default::default()
},
];
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
assert!(
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
"site replication add reported failure: {add_status:?}"
);
wait_for_site_replication_enabled(&site_a.cluster).await?;
wait_for_site_replication_enabled(&site_b.cluster).await?;
let info_a = site_replication_info(&site_a.cluster).await?;
let remote = info_a
.sites
.iter()
.find(|site| site.name == "site-b")
.ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
assert!(
deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
"site peers must have two distinct non-empty deployment IDs: {info_a:?}"
);
assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
let key = "site-object.bin";
let body = b"four-node-site-replication".to_vec();
put_object(&client_a, &bucket, key, body.clone()).await?;
wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
let peer_b = site_b.client(3)?;
wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
let reverse_key = "reverse/site-object.bin";
let reverse_body = b"site-b-to-site-a".to_vec();
put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
Ok(())
}
@@ -0,0 +1,345 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! 4-node upgrade coverage for historical objects and IAM AK/SK.
//!
//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
//! mixed-version listing). This module pins the distributed contract the
//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
//! written on the previous release still read back, and IAM user credentials
//! created before the upgrade still authenticate.
//!
//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
//! release. The `e2e-distributed` workflow downloads that binary; a local run
//! without it fails closed rather than skipping.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
unique_bucket, wait_until,
};
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
use std::time::Duration;
use uuid::Uuid;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const IAM_SECRET: &str = "UpgradeTestSecretKey1";
const WRONG_SECRET: &str = "WrongSecretKey000000";
const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
struct UpgradeSeed {
history_bucket: String,
history_key: &'static str,
history_body: Vec<u8>,
versioned_bucket: String,
versioned_key: &'static str,
version1: String,
version1_body: Vec<u8>,
version2: String,
version2_body: Vec<u8>,
iam_bucket: String,
iam_key: &'static str,
iam_body: Vec<u8>,
iam_user: String,
iam_secret: &'static str,
}
fn source_binary() -> TestResult<PathBuf> {
let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
format!(
"{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
)
})?;
if !path.is_file() {
return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
}
Ok(path)
}
fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
return Ok(());
};
std::fs::create_dir_all(&log_dir)?;
for node_idx in 0..cluster.cluster.nodes.len() {
let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
cluster
.cluster
.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
}
Ok(())
}
fn iam_rw_policy(bucket: &str) -> String {
serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:*"],
"Resource": [
format!("arn:aws:s3:::{bucket}"),
format!("arn:aws:s3:::{bucket}/*")
]
}]
})
.to_string()
}
async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
let url = &dist.cluster.nodes[0].url;
let access = &dist.cluster.access_key;
let admin_secret = &dist.cluster.secret_key;
admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
Ok(())
}
async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
wait_until(
CREDENTIAL_TIMEOUT,
|| {
let client = client.clone();
let bucket = bucket.to_string();
let key = key.to_string();
let body = body.clone();
async move {
put_object(&client, &bucket, &key, body).await?;
Ok(true)
}
},
label,
)
.await
}
async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
wait_until(
CREDENTIAL_TIMEOUT,
|| {
let client = client.clone();
let bucket = bucket.to_string();
let key = key.to_string();
let expected = expected.to_vec();
async move {
let got = get_object_bytes(&client, &bucket, &key).await?;
Ok(got == expected)
}
},
label,
)
.await
}
async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
let history_bucket = unique_bucket("upg-hist");
let versioned_bucket = unique_bucket("upg-ver");
let iam_bucket = unique_bucket("upg-iam");
dist.create_bucket(&history_bucket).await?;
dist.create_bucket(&versioned_bucket).await?;
dist.create_bucket(&iam_bucket).await?;
let root = dist.client(0)?;
enable_versioning(&root, &versioned_bucket).await?;
let history_key = "plain-history.bin";
let history_body = b"written by the previous 4-node release".to_vec();
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
let versioned_key = "versioned-history.txt";
let version1_body = b"version-one-before-upgrade".to_vec();
let version1 = root
.put_object()
.bucket(&versioned_bucket)
.key(versioned_key)
.body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
.send()
.await?
.version_id()
.ok_or("first versioned PUT omitted version ID")?
.to_string();
let version2_body = b"version-two-before-upgrade".to_vec();
let version2 = root
.put_object()
.bucket(&versioned_bucket)
.key(versioned_key)
.body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
.send()
.await?
.version_id()
.ok_or("second versioned PUT omitted version ID")?
.to_string();
let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
let iam_key = "iam-history.bin";
let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
Ok(UpgradeSeed {
history_bucket,
history_key,
history_body,
versioned_bucket,
versioned_key,
version1,
version1_body,
version2,
version2_body,
iam_bucket,
iam_key,
iam_body,
iam_user,
iam_secret: IAM_SECRET,
})
}
async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
let root_a = dist.client(0)?;
let root_b = dist.client(3)?;
wait_for_bytes(
&root_b,
&seed.history_bucket,
seed.history_key,
&seed.history_body,
&format!("{context}: root GET historical object"),
)
.await?;
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
let v1 = root_b
.get_object()
.bucket(&seed.versioned_bucket)
.key(seed.versioned_key)
.version_id(&seed.version1)
.send()
.await?;
let v1_body = v1.body.collect().await?.into_bytes();
if v1_body.as_ref() != seed.version1_body.as_slice() {
return Err(format!("{context}: version 1 bytes changed after upgrade").into());
}
let v2 = root_a
.get_object()
.bucket(&seed.versioned_bucket)
.key(seed.versioned_key)
.version_id(&seed.version2)
.send()
.await?;
let v2_body = v2.body.collect().await?.into_bytes();
if v2_body.as_ref() != seed.version2_body.as_slice() {
return Err(format!("{context}: version 2 bytes changed after upgrade").into());
}
let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
if !users.contains(&seed.iam_user) {
return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
}
let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
wait_for_bytes(
&iam_on_upgraded,
&seed.iam_bucket,
seed.iam_key,
&seed.iam_body,
&format!("{context}: IAM GET historical object on node 0"),
)
.await?;
wait_for_bytes(
&iam_on_peer,
&seed.iam_bucket,
seed.iam_key,
&seed.iam_body,
&format!("{context}: IAM GET historical object on node 3"),
)
.await?;
let post_key = format!("after-upgrade-{context}.txt");
let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
wait_for_put(
&iam_on_peer,
&seed.iam_bucket,
&post_key,
post_body.clone(),
&format!("{context}: IAM PUT after upgrade"),
)
.await?;
assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
Err(error) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
let rejected = code == Some("SignatureDoesNotMatch")
|| code == Some("InvalidAccessKeyId")
|| code == Some("AccessDenied")
|| code == Some("InvalidArgument")
|| error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
if !rejected {
return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
}
}
}
let post_root_key = format!("root-after-{context}.bin");
let post_root_body = format!("{context}: root write after upgrade").into_bytes();
put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
Ok(())
}
#[tokio::test]
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
dist.start_from_binary(&previous).await?;
let seed = seed_history_and_iam(&dist).await?;
dist.restart_with_current_binary().await?;
assert_history_and_iam(&dist, &seed, "direct").await?;
Ok(())
}
#[tokio::test]
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
dist.start_from_binary(&previous).await?;
let seed = seed_history_and_iam(&dist).await?;
dist.replace_node_with_current_binary(0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
for node_idx in [1, 2] {
dist.replace_node_with_current_binary(node_idx).await?;
}
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
dist.replace_node_with_current_binary(3).await?;
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
Ok(())
}
@@ -0,0 +1,188 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
#[tokio::test]
async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("version");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
enable_versioning(&writer, &bucket).await?;
let key = "versioned.txt";
let v1_id = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"v1".to_vec().into())
.send()
.await?
.version_id()
.ok_or("v1 PUT omitted version ID")?
.to_string();
let v2_id = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"v2".to_vec().into())
.send()
.await?
.version_id()
.ok_or("v2 PUT omitted version ID")?
.to_string();
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching_versions: Vec<_> = versions
.versions()
.iter()
.filter(|version| version.key() == Some(key))
.collect();
assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
assert!(
matching_versions
.iter()
.any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
"v1 was not the historical version: {versions:?}"
);
assert!(
matching_versions
.iter()
.any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
"v2 was not the latest version: {versions:?}"
);
let latest = get_object_bytes(&reader, &bucket, key).await?;
assert_eq!(latest, b"v2");
let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
let older_body = older.body.collect().await?.into_bytes();
assert_eq!(older_body.as_ref(), b"v1");
let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
assert_eq!(deleted.delete_marker(), Some(true));
let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching_markers: Vec<_> = after_delete
.delete_markers()
.iter()
.filter(|marker| marker.key() == Some(key))
.collect();
assert_eq!(
matching_markers.len(),
1,
"delete marker missing or duplicated after current-version delete: {after_delete:?}"
);
assert!(
matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
"DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
);
let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
match latest_after_delete {
Ok(_) => return Err("current version should be a delete marker".into()),
Err(error)
if error
.as_service_error()
.and_then(ProvideErrorMetadata::code)
.is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
Err(error) => return Err(error.into()),
}
let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
let restored_body = restored.body.collect().await?.into_bytes();
assert_eq!(restored_body.as_ref(), b"v1");
writer
.delete_object()
.bucket(&bucket)
.key(key)
.version_id(marker_id)
.send()
.await?;
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
Ok(())
}
#[tokio::test]
async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
init_logging();
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("suspend");
dist.create_bucket(&bucket).await?;
let writer = dist.client(0)?;
let reader = dist.client(3)?;
enable_versioning(&writer, &bucket).await?;
let key = "suspended.txt";
let original = writer
.put_object()
.bucket(&bucket)
.key(key)
.body(b"enabled-history".to_vec().into())
.send()
.await?
.version_id()
.ok_or("enabled PUT omitted version ID")?
.to_string();
writer
.put_bucket_versioning()
.bucket(&bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Suspended)
.build(),
)
.send()
.await?;
put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
let matching: Vec<_> = versions
.versions()
.iter()
.filter(|version| version.key() == Some(key))
.collect();
assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
let null_version_count = matching
.iter()
.filter(|version| {
matches!(
version.version_id(),
None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
)
})
.count();
assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
let historical = reader
.get_object()
.bucket(&bucket)
.key(key)
.version_id(&original)
.send()
.await?;
assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
Ok(())
}
@@ -16,15 +16,18 @@
#[cfg(test)]
mod tests {
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
use crate::common::{
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
rustfs_binary_path,
};
use crate::storage_api::RUSTFS_META_BUCKET;
use aws_sdk_s3::primitives::ByteStream;
use http::Method;
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::Command;
@@ -34,6 +37,76 @@ mod tests {
const POOL_METADATA_OBJECT: &str = "pool.bin";
#[derive(serde::Deserialize)]
struct EvidenceBuild {
sha256: String,
}
#[derive(serde::Deserialize)]
struct RestartEvidenceRun {
schema: u32,
run_id: String,
source_revision: String,
test_build: serde_json::Value,
binary: EvidenceBuild,
test_binary: EvidenceBuild,
}
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
let mut file = std::fs::File::open(path)?;
let mut digest = Sha256::new();
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
digest.update(&buffer[..read]);
}
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
}
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
return Ok(None);
};
let directory = PathBuf::from(directory);
let receipt = directory.join("run.json");
if receipt.metadata()?.len() > 1024 * 1024 {
return Err("oversized scanner/heal execution receipt".into());
}
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
return Err("invalid scanner/heal execution identity".into());
}
let built = compiled_test_identity();
for key in ["source_revision", "dirty", "lock_blob", "features"] {
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
}
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
assert_eq!(
file_sha256(&std::env::current_exe()?)?,
run.test_binary.sha256,
"test executable must match the run receipt"
);
if directory.join("background-target-restart.json").exists() {
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
}
Ok(Some((directory, run)))
}
fn compiled_test_identity() -> serde_json::Value {
serde_json::json!({
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
})
}
struct TcpPortBlackhole {
port: u16,
comment: String,
@@ -195,8 +268,9 @@ mod tests {
clients: &[aws_sdk_s3::Client],
bucket: &str,
expected_keys: &HashSet<String>,
) -> Result<(), Box<dyn Error + Send + Sync>> {
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
const PAGE_SIZE: i32 = 10;
let mut node_listings = Vec::with_capacity(clients.len());
for (node_index, client) in clients.iter().enumerate() {
let mut listed_keys = Vec::new();
let mut continuation_token = None;
@@ -243,8 +317,10 @@ mod tests {
&listed_key_set, expected_keys,
"node {node_index} did not expose the complete recovered namespace"
);
listed_keys.sort();
node_listings.push(listed_keys);
}
Ok(())
Ok(node_listings)
}
fn heal_task_status_diagnostic(body: &str) -> String {
@@ -808,6 +884,13 @@ mod tests {
}
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
restart_evidence_run(&server_binary)?
} else {
None
};
let mut evidence_objects = Vec::new();
let (background_enabled, interruption_node, interruption_kind) = match scenario {
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
@@ -855,7 +938,7 @@ mod tests {
for node_index in 0..cluster.nodes.len() {
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
}
cluster.start().await?;
cluster.start_with_binary(&server_binary).await?;
let clients = cluster.create_all_clients()?;
let bucket = "heal-restart-during-rebuild";
@@ -996,7 +1079,7 @@ mod tests {
}
}
cluster.start_node(1).await?;
cluster.start_node_from_binary(1, &server_binary).await?;
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1274,7 +1357,7 @@ mod tests {
}
}
}
cluster.start_node(interruption_node).await?;
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
if interruption_node == 0 {
let target = cluster.nodes[1]
.process
@@ -1373,7 +1456,7 @@ mod tests {
.map(|manifest| manifest.key.clone())
.collect::<HashSet<_>>();
assert!(expected_keys.insert(outage_key.to_string()));
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
let target_client = cluster.create_s3_client(1)?;
for expected in &expected_manifests {
@@ -1381,11 +1464,31 @@ mod tests {
let actual = response.body.collect().await?.into_bytes();
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
if evidence_run.is_some() {
evidence_objects.push(serde_json::json!({
"key": expected.key, "version_id": expected.shard_census.version_id,
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
"expected_sha256": sha256_hex(&expected_body),
"actual_sha256": sha256_hex(&actual),
"expected_physical": expected.shard_census,
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
}));
}
}
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
let actual = response.body.collect().await?.into_bytes();
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
if evidence_run.is_some() {
evidence_objects.push(serde_json::json!({
"key": outage_key, "version_id": null,
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
"expected_sha256": sha256_hex(&expected_outage_body),
"actual_sha256": sha256_hex(&actual),
"expected_physical": null,
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
}));
}
let terminal_deadline = Instant::now() + Duration::from_secs(30);
loop {
@@ -1432,6 +1535,31 @@ mod tests {
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
}
if let Some((directory, run)) = evidence_run {
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
assert_ne!(target_pid, restarted_pid, "target must be a new process");
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
let evidence = serde_json::json!({
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
"run_id": run.run_id, "source_revision": run.source_revision,
"test_build": compiled_test_identity(),
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
"pid_before": target_pid, "pid_after": restarted_pid,
"objects": evidence_objects, "node_listings": node_listings,
});
let data = serde_json::to_vec(&evidence)?;
if data.len() > 1024 * 1024 {
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
}
let mut output = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(directory.join("background-target-restart.json"))?;
output.write_all(&data)?;
output.sync_all()?;
}
Ok(())
}
+5
View File
@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
#[cfg(test)]
mod distributed_startup_regression_test;
// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
// quota, observability, expand/decommission/rebalance, site replication, chaos).
#[cfg(test)]
mod distributed;
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
#[cfg(test)]
mod tier_transition_regression_test;
+11 -3
View File
@@ -32,7 +32,7 @@ pub mod bucket {
pub mod bucket_target_sys {
pub use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
SsecPassthroughCapability, TargetClient, append_version_id_query,
SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query,
};
}
@@ -69,6 +69,13 @@ pub mod bucket {
};
}
pub mod recovery_control {
pub use crate::bucket::lifecycle::recovery_control::{
IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
inspect_recovery_control, list_recovery_controls,
};
}
pub mod transition_transaction {
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
@@ -89,8 +96,9 @@ pub mod bucket {
#[allow(clippy::module_inception)]
pub mod lifecycle {
pub use crate::bucket::lifecycle::lifecycle::{
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
object_opts_from_object_info,
};
}
+121 -5
View File
@@ -369,6 +369,26 @@ struct SsecPassthroughRecord {
recorded_at: Instant,
}
/// What a target write does when the bucket's persisted target set exists but
/// cannot be decoded.
///
/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a
/// configuration that could not be fully read, because re-serializing a
/// partial in-memory view is the one mechanism by which a configured target
/// really disappears. That rule guards against an *unintentional* overwrite,
/// so an operator who names the hazard keeps a repair path
/// (rustfs/backlog#2309); everything that does not name it stays refused.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum UnreadableTargetsPolicy {
/// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`].
#[default]
FailClosed,
/// Discard the unreadable set; the target being written becomes the whole
/// configuration. Reachable only from an admin request that asked for it
/// explicitly, and audited by the caller.
Replace,
}
#[derive(Debug, Default)]
pub struct BucketTargetSys {
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
@@ -791,20 +811,45 @@ impl BucketTargetSys {
bucket: &str,
target: &BucketTarget,
update: bool,
unreadable_policy: UnreadableTargetsPolicy,
) -> Result<BucketTargets, BucketTargetError> {
self.validate_target(bucket, target).await?;
let mut bucket_targets = match self.list_bucket_targets(bucket).await {
Ok(targets) => targets,
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
Err(err) => return Err(err),
};
let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?;
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
Ok(bucket_targets)
}
/// The persisted target set a write merges into.
///
/// An absent configuration starts from the empty set. An unreadable one is
/// refused, because re-serializing a partial view of a set this node could
/// not decode is how a configured target disappears for good — unless the
/// caller carries the operator's explicit
/// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it
/// deliberately (rustfs/backlog#2309).
async fn targets_base_for_write(
&self,
bucket: &str,
unreadable_policy: UnreadableTargetsPolicy,
) -> Result<BucketTargets, BucketTargetError> {
match self.list_bucket_targets(bucket).await {
Ok(targets) => Ok(targets),
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
// The opt-in discards only a set this node genuinely cannot read.
// A readable set still merges through the arm above, so the policy
// can never drop a target that was visible here.
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
if unreadable_policy == UnreadableTargetsPolicy::Replace =>
{
Ok(BucketTargets::default())
}
Err(err) => Err(err),
}
}
pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
if !target.target_type.is_valid() {
return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
@@ -4313,4 +4358,75 @@ mod tests {
let window = LastMinuteLatency::new();
assert_eq!(window.get_total().avg, Duration::from_secs(0));
}
fn repair_target(bucket: &str, id: &str) -> BucketTarget {
BucketTarget {
source_bucket: bucket.to_string(),
endpoint: "remote.example.com".to_string(),
target_bucket: "remote".to_string(),
arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
target_type: BucketTargetType::ReplicationService,
region: "us-east-1".to_string(),
..Default::default()
}
}
/// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable
/// `bucket-targets.json` left the bucket with no API repair path at all.
/// The refusal is the default and stays the default; the operator's
/// explicit opt-in is the only thing that discards the set, and it starts
/// the replacement from empty rather than from a partial view of bytes
/// this node never decoded.
#[tokio::test]
async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() {
let sys = BucketTargetSys::default();
let bucket = "targets-repair-opt-in";
sys.mark_targets_unreadable(bucket).await;
assert!(
matches!(
sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
),
"without the opt-in an unreadable target set must still refuse the write"
);
assert_eq!(
UnreadableTargetsPolicy::default(),
UnreadableTargetsPolicy::FailClosed,
"a caller that says nothing must get the refusal"
);
let base = sys
.targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace)
.await
.expect("the explicit opt-in must let an operator replace an unreadable set");
assert!(
base.is_empty(),
"the replacement must start from an empty set, never from a partial decode"
);
}
/// The opt-in is not a wipe switch. On a set this node can read, both
/// policies take the same merge path, so a stray `replace-unreadable=true`
/// cannot drop a visible target — which is what makes the flag safe to
/// repeat in an operator's repair script.
#[tokio::test]
async fn the_opt_in_never_discards_a_readable_target_set() {
let sys = BucketTargetSys::default();
let bucket = "targets-repair-readable";
let existing = repair_target(bucket, "keep");
sys.targets_map
.write()
.await
.insert(bucket.to_string(), vec![existing.clone()]);
for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] {
let base = sys
.targets_base_for_write(bucket, policy)
.await
.expect("a readable target set must be readable under either policy");
assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target");
assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target");
}
}
}
@@ -150,6 +150,18 @@ static XXHASH_SEED: u64 = 0;
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
#[cfg(test)]
#[derive(Default)]
struct FreeVersionPostRemoteDeleteTestBarrier {
arrived: Notify,
release: Notify,
}
#[cfg(test)]
tokio::task_local! {
static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc<FreeVersionPostRemoteDeleteTestBarrier>;
}
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
#[allow(
dead_code,
@@ -910,6 +922,11 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
})??;
}
}
#[cfg(test)]
if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
// Remote DELETE is idempotent, but a changed fence makes the local
// outcome ambiguous. Keep every marker for a fully fenced retry.
@@ -5831,7 +5848,7 @@ mod tests {
#[cfg(feature = "test-util")]
use crate::services::tier::test_util::register_mock_tier;
#[cfg(feature = "test-util")]
use crate::services::tier::tier::TierConfigMgr;
use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory};
#[cfg(feature = "test-util")]
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
@@ -7830,6 +7847,119 @@ mod tests {
}
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial]
async fn tier_remove_waits_for_inflight_free_version_local_commit() {
let (disk_paths, ecstore) = setup_test_env().await;
let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4());
let object = "free-version";
create_test_bucket(&ecstore, &bucket).await;
let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await;
let tier_manager = ecstore.tier_config_mgr();
{
let manager = tier_manager.read().await;
manager
.save_tiering_config(Arc::clone(&ecstore))
.await
.expect("mock tier configuration should persist before removal");
}
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await;
let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
.await
.expect("seeded free version should be listed");
let oi = page
.items
.into_iter()
.next()
.expect("seeded free version should be recoverable");
backend
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
.await;
let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM")
.await
.expect("mock tier lease should be available");
seed_lease
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
.await
.expect("remote free-version tuple should be seeded");
drop(seed_lease);
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
let cleanup_barrier = Arc::clone(&barrier);
let cleanup_store = Arc::clone(&ecstore);
let cleanup_oi = oi.clone();
let cleanup = tokio::spawn(async move {
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
.scope(cleanup_barrier, async move {
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
})
.await
});
tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified())
.await
.expect("free-version cleanup should pause after the remote delete");
assert!(!backend.contains(&oi.transitioned_object.name).await);
let remove_manager = Arc::clone(&tier_manager);
let remove_store = Arc::clone(&ecstore);
let remove_backend = backend.clone();
let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone())));
let mut remove = tokio::spawn(async move {
TIER_DRIVER_TEST_FACTORY
.scope(
remove_driver_factory,
TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true),
)
.await
});
let prepared = tokio::time::timeout(StdDuration::from_secs(30), async {
loop {
match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await {
Ok(lease) => drop(lease),
Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break,
Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"),
}
tokio::task::yield_now().await;
}
});
tokio::select! {
prepared = prepared => {
prepared.expect("tier remove should install its prepared admission fence");
}
result = &mut remove => {
panic!("tier remove finished before installing its prepared admission fence: {result:?}");
}
}
assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit");
barrier.release.notify_one();
tokio::time::timeout(StdDuration::from_secs(30), cleanup)
.await
.expect("free-version cleanup should finish after release")
.expect("free-version cleanup task should join")
.expect("free-version cleanup should keep its generation current");
tokio::time::timeout(StdDuration::from_secs(30), remove)
.await
.expect("tier remove should finish after local cleanup")
.expect("tier remove task should join")
.expect("tier remove should pass its fresh authoritative proof");
assert!(!tier_manager.read().await.is_tier_valid("WARM"));
for disk_path in &disk_paths {
assert!(
!fs::try_exists(disk_path.join(&bucket).join(object))
.await
.expect("post-removal free-version path check should succeed")
);
}
ecstore
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("empty free-version test bucket should be removed");
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial]
+3 -3
View File
@@ -15,9 +15,9 @@
use crate::object_api::ObjectInfo;
pub use rustfs_lifecycle::{
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
expiration_action_has_valid_target,
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
expected_expiry_time, expiration_action_has_valid_target,
};
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
@@ -22,7 +22,7 @@ use super::{
bucket_lifecycle_ops::{
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
},
manual_transition_job, tier_delete_journal, transition_transaction,
manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
};
use crate::error::{Error, Result};
use crate::services::tier::tier_probe_intent;
@@ -41,6 +41,7 @@ pub(crate) enum DurableIlmRecordKind {
ManualTransitionScope,
ManualTransitionTask,
ManualTransitionWorkerResult,
RecoveryControl,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -105,8 +106,14 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
};
pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
name: "recovery-control",
prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
kind: DurableIlmRecordKind::RecoveryControl,
};
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
TIER_DELETE_JOURNAL_NAMESPACE,
TIER_DELETE_JOURNAL_V6_NAMESPACE,
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
@@ -116,6 +123,7 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
MANUAL_TRANSITION_SCOPE_NAMESPACE,
MANUAL_TRANSITION_TASK_NAMESPACE,
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
RECOVERY_CONTROL_NAMESPACE,
];
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -241,6 +249,18 @@ pub(crate) enum DurableIlmRecordCheckpoint {
ManualTransitionWorkerResult {
content_sha256: String,
},
RecoveryControl {
content_sha256: String,
identity_sha256: String,
source_generation_sha256: String,
first_seen_at_unix_nanos: i64,
revision: u64,
classification: recovery_control::IlmRecoveryClassification,
attempt_count: u64,
consecutive_failure_count: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
owner_fence_sha256: Option<String>,
},
}
impl DurableIlmRecordCheckpoint {
@@ -254,7 +274,8 @@ impl DurableIlmRecordCheckpoint {
| Self::ManualTransitionJob { content_sha256, .. }
| Self::ManualTransitionScope { content_sha256, .. }
| Self::ManualTransitionTask { content_sha256 }
| Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
| Self::ManualTransitionWorkerResult { content_sha256 }
| Self::RecoveryControl { content_sha256, .. } => content_sha256,
}
}
@@ -528,6 +549,51 @@ impl DurableIlmRecordCheckpoint {
..
},
) => previous_identity == next_identity && next_updated_at > previous_updated_at,
(
Self::RecoveryControl {
identity_sha256: previous_identity,
source_generation_sha256: previous_generation,
first_seen_at_unix_nanos: previous_first_seen,
revision: previous_revision,
classification: previous_classification,
attempt_count: previous_attempts,
consecutive_failure_count: previous_failures,
owner_fence_sha256: previous_owner,
..
},
Self::RecoveryControl {
identity_sha256: next_identity,
source_generation_sha256: next_generation,
first_seen_at_unix_nanos: next_first_seen,
revision: next_revision,
classification: next_classification,
attempt_count: next_attempts,
consecutive_failure_count: next_failures,
owner_fence_sha256: next_owner,
..
},
) => {
let adjacent = previous_identity == next_identity
&& previous_first_seen == next_first_seen
&& previous_revision.checked_add(1) == Some(*next_revision);
let claim = next_owner.is_some()
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
&& previous_attempts.checked_add(1) == Some(*next_attempts)
&& previous_failures == next_failures;
let source_refresh = previous_owner.is_some()
&& previous_owner == next_owner
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
&& previous_attempts == next_attempts
&& previous_failures == next_failures
&& previous_generation != next_generation;
let completion = previous_owner.is_some()
&& next_owner.is_none()
&& previous_generation == next_generation
&& previous_attempts == next_attempts;
adjacent && (claim || source_refresh || completion)
}
_ => false,
};
@@ -553,6 +619,14 @@ impl DurableIlmRecordCheckpoint {
{
return false;
}
if let Self::RecoveryControl { classification, .. } = terminal
&& !matches!(
classification,
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
)
{
return false;
}
if self == terminal || self.validate_successor(terminal).is_ok() {
return true;
}
@@ -652,6 +726,32 @@ impl DurableIlmRecordCheckpoint {
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
}
(
Self::RecoveryControl {
identity_sha256: previous_identity,
source_generation_sha256: previous_generation,
first_seen_at_unix_nanos: previous_first_seen,
revision: previous_revision,
attempt_count: previous_attempts,
..
},
Self::RecoveryControl {
identity_sha256: terminal_identity,
source_generation_sha256: terminal_generation,
first_seen_at_unix_nanos: terminal_first_seen,
revision: terminal_revision,
attempt_count: terminal_attempts,
classification:
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
..
},
) => {
previous_identity == terminal_identity
&& (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
&& previous_first_seen == terminal_first_seen
&& terminal_revision > previous_revision
&& terminal_attempts >= previous_attempts
}
_ => false,
}
}
@@ -1219,6 +1319,35 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
},
)
}
DurableIlmRecordKind::RecoveryControl => {
let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
.map_err(|err| Error::other(err.to_string()))?;
let control =
recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
.map_err(|err| Error::other(err.to_string()))?;
if canonical != path || control.identity.protocol != protocol {
return Err(Error::other("ILM recovery control path is not canonical"));
}
let identity_sha256 = checkpoint_hash(&control.identity)?;
let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
(
"control_id",
control_id,
DurableIlmRecordCheckpoint::RecoveryControl {
content_sha256,
identity_sha256,
source_generation_sha256,
first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
revision: control.revision,
classification: control.classification,
attempt_count: control.attempt_count,
consecutive_failure_count: control.consecutive_failure_count,
owner_fence_sha256,
},
)
}
DurableIlmRecordKind::ManualTransitionJob => {
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
.map_err(|err| Error::other(err.to_string()))?;
@@ -1412,6 +1541,94 @@ mod tests {
.checkpoint
}
fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
let generation = recovery_control::IlmRecoverySourceGeneration::new(
transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
"source-etag",
"a".repeat(64),
vec![recovery_control::IlmRecoverySourceCopy {
authority: "pool-0/set-0".to_string(),
canonical_path: source_path.to_string(),
etag: "source-etag".to_string(),
encoded_len: 128,
content_sha256: "a".repeat(64),
}],
)
.expect("source generation should build");
recovery_control::IlmRecoveryControl::new(
recovery_control::IlmRecoveryControlIdentity {
protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
canonical_source_path: source_path.to_string(),
stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
record_class: "transition_transaction_v1".to_string(),
},
generation,
recovery_control::IlmRecoveryClassification::Retrying,
1_000_000_000,
recovery_control::IlmRecoveryErrorCode::None,
)
.expect("recovery control should build")
}
fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
let control_id = control.identity.source_operation_digest().expect("control id should derive");
let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
.expect("control path should build");
let encoded = control.encode().expect("control should encode");
let namespace = classify_durable_ilm_record(&path)
.expect("recovery control namespace should classify")
.expect("recovery control should be durable");
assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
validate_durable_ilm_record(&path, &encoded)
.expect("recovery control should validate")
.checkpoint
}
#[test]
fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
let initial_control = recovery_control_fixture();
let initial = recovery_control_checkpoint(&initial_control);
let mut claimed_control = initial_control;
let mut advanced_generation = claimed_control.observed_source_generation.clone();
advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
claimed_control
.claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
.expect("control should claim");
let claimed = recovery_control_checkpoint(&claimed_control);
initial.validate_successor(&claimed).expect("claim should advance receipt");
let mut retry_control = claimed_control;
retry_control
.record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
.expect("retry should persist");
let retry = recovery_control_checkpoint(&retry_control);
claimed.validate_successor(&retry).expect("retry should advance receipt");
let ready_at = retry_control
.next_attempt_at_unix_nanos
.expect("retry deadline should persist");
let mut terminal_control = retry_control;
terminal_control
.claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
.expect("retry should claim");
let reclaimed = recovery_control_checkpoint(&terminal_control);
retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
terminal_control
.finish_attempt(
recovery_control::IlmRecoveryClassification::Terminal,
recovery_control::IlmRecoveryErrorCode::None,
)
.expect("control should terminate");
let terminal = recovery_control_checkpoint(&terminal_control);
reclaimed
.validate_successor(&terminal)
.expect("terminal state should advance receipt");
assert!(initial.is_predecessor_of_terminal(&terminal));
assert!(!initial.is_predecessor_of_terminal(&retry));
}
#[test]
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
let initial_intent = tier_probe_intent_fixture();
@@ -24,6 +24,7 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
mod object_handlers_common;
mod object_lock_boundary;
pub use self::core as lifecycle;
pub mod recovery_control;
mod replication_sink;
pub mod rule;
mod runtime_boundary;
File diff suppressed because it is too large Load Diff
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
#[cfg(test)]
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
obj_name: &str,
rv_id: &str,
@@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
#[cfg(test)]
mod test {
#[cfg(feature = "test-util")]
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
use super::{
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use rustfs_filemeta::TransitionVersionState;
@@ -23,6 +23,11 @@ use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::recovery_control::{
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
save_recovery_control_if_absent, save_recovery_control_if_current,
};
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
@@ -44,6 +49,7 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
@@ -737,9 +743,11 @@ pub enum TransitionTransactionRecoveryOutcome {
RemoteCandidateDeleted,
RecordDeleted,
Retained,
RetainedAmbiguous(IlmRecoveryErrorCode),
OperatorRequired(IlmRecoveryErrorCode),
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
#[derive(Default)]
struct TransitionRecoveryClaimBarrierState {
transaction_id: Uuid,
@@ -747,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionRecoveryClaimBarrier {
state: Arc<TransitionRecoveryClaimBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl TransitionRecoveryClaimBarrier {
pub(crate) fn install(transaction_id: Uuid) -> Self {
let state = Arc::new(TransitionRecoveryClaimBarrierState {
@@ -788,7 +796,7 @@ impl TransitionRecoveryClaimBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionRecoveryClaimBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -802,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -817,6 +825,80 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
}
}
#[cfg(all(test, feature = "test-util"))]
#[derive(Default)]
struct TransitionRecoveryTerminalBarrierState {
transaction_id: Uuid,
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionRecoveryTerminalBarrier {
state: Arc<TransitionRecoveryTerminalBarrierState>,
}
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(all(test, feature = "test-util"))]
impl TransitionRecoveryTerminalBarrier {
pub(crate) fn install(transaction_id: Uuid) -> Self {
let state = Arc::new(TransitionRecoveryTerminalBarrierState {
transaction_id,
..Default::default()
});
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("transition recovery terminal barrier mutex should not poison");
assert!(
slot.is_none(),
"transition recovery terminal barrier must be installed by one test at a time"
);
*slot = Some(Arc::clone(&state));
drop(slot);
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("transition recovery should persist terminal control before source cleanup");
}
}
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionRecoveryTerminalBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("transition recovery terminal barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
}
}
#[cfg(all(test, feature = "test-util"))]
async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("transition recovery terminal barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.transaction_id == transaction_id)
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TransitionOperatorProbe {
@@ -1020,17 +1102,35 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
let suffix = object
.strip_prefix(&prefix)
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
let file_name = suffix
.rsplit('/')
let mut parts = suffix.split('/');
let shard_a = parts
.next()
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
let shard_b = parts
.next()
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
let file_name = parts
.next()
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
if parts.next().is_some() {
return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
}
let transaction_key = file_name
.strip_suffix(".json")
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
if transaction_key.len() != 32
|| !transaction_key
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|| shard_a != &transaction_key[..2]
|| shard_b != &transaction_key[2..4]
{
return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
}
Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
Uuid::parse_str(transaction_key)
.ok()
.filter(|transaction_id| !transaction_id.is_nil())
.ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
}
pub async fn process_transition_transaction_record(
@@ -1055,6 +1155,27 @@ async fn process_transition_transaction_record_at(
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
let record_name =
transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
let now_unix_nanos =
i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
let recovery_control_id = recovery_control_identity
.source_operation_digest()
.map_err(|err| Error::other(err.to_string()))?;
let control_record_name =
recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
.map_err(|err| Error::other(err.to_string()))?;
let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
Some(
api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
.await?,
)
} else {
None
};
let _control_guard = match &control_lock {
Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
None => None,
};
// The synthetic key avoids nesting the recovery lock with the config
// object's own I/O lock. Holding it across the bounded source proof and
// remote DELETE elects one destructive recovery worker across nodes.
@@ -1073,55 +1194,400 @@ async fn process_transition_transaction_record_at(
return Ok(TransitionTransactionRecoveryOutcome::Retained);
}
match current.state {
let mut recovery_control = if transition_state_needs_recovery_control(&current, now_unix_nanos) {
if cleanup_terminal_transition_recovery_control(
api.clone(),
&current,
&record_name,
&recovery_control_identity,
&recovery_control_id,
)
.await?
{
return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
}
match claim_transition_recovery_control(
api.clone(),
&current,
&record_name,
recovery_control_identity,
&recovery_control_id,
now_unix_nanos,
)
.await?
{
Some(control) => Some(control),
None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
}
} else {
None
};
let recovery = match current.state {
TransitionTransactionState::Uploaded => {
if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
return Ok(TransitionTransactionRecoveryOutcome::Retained);
}
let mut cleanup = current.clone();
cleanup
.mark_cleanup_pending(
current.fence(),
TransitionCleanupProof {
transaction_id: current.transaction_id,
write_id: current.write_id,
remote_object: current.remote_object.clone(),
remote_version: current.remote_version.clone(),
backend_fingerprint: current.backend_fingerprint,
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
},
)
.map_err(transition_transaction_store_error)?;
#[cfg(test)]
pause_before_transition_recovery_claim(current.transaction_id).await;
match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
Ok(()) => recover_cleanup_pending(api, &cleanup).await,
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
Err(err) => Err(err),
if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
Ok(TransitionTransactionRecoveryOutcome::Retained)
} else {
let mut cleanup = current.clone();
cleanup
.mark_cleanup_pending(
current.fence(),
TransitionCleanupProof {
transaction_id: current.transaction_id,
write_id: current.write_id,
remote_object: current.remote_object.clone(),
remote_version: current.remote_version.clone(),
backend_fingerprint: current.backend_fingerprint,
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
},
)
.map_err(transition_transaction_store_error)?;
#[cfg(all(test, feature = "test-util"))]
pause_before_transition_recovery_claim(current.transaction_id).await;
match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
Ok(TransitionTransactionRecoveryOutcome::Retained)
}
Err(err) => Err(err),
}
}
}
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, &current).await,
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), &current).await,
TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), &current).await {
Ok(true) => {
delete_transition_transaction_record(api, &current).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
IlmRecoveryErrorCode::LocalCommitAmbiguous,
)),
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
IlmRecoveryErrorCode::LocalCommitAmbiguous,
)),
Err(err) => Err(err),
},
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
delete_transition_transaction_record(api, &current).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionTransactionState::UploadOutcomeUnknown => {
if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
Ok(TransitionTransactionRecoveryOutcome::Retained)
} else {
recover_unknown_upload_outcome(api, &current).await
recover_unknown_upload_outcome(api.clone(), &current).await
}
}
TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
TransitionTransactionState::UploadStarted => {
if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
Ok(TransitionTransactionRecoveryOutcome::Retained)
} else {
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
IlmRecoveryErrorCode::RemoteVersionUnknown,
))
}
}
};
if let Some(mut control) = recovery_control.take() {
let source_to_delete = if matches!(
recovery,
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
| TransitionTransactionRecoveryOutcome::RecordDeleted)
) {
let refreshed =
refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
control = refreshed.0;
refreshed.1
} else {
None
};
persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
if let Some(source) = source_to_delete {
#[cfg(all(test, feature = "test-util"))]
pause_after_transition_recovery_terminal(source.transaction_id).await;
delete_transition_transaction_record(api, &source).await?;
}
} else if matches!(
recovery,
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted)
) {
delete_transition_transaction_record(api, &current).await?;
}
recovery
}
fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
IlmRecoveryControlIdentity {
protocol: IlmRecoveryProtocol::TransitionTransaction,
canonical_source_path: record_name.to_string(),
stable_operation_identity: transaction.transaction_id.to_string(),
record_class: "transition_transaction_v1".to_string(),
}
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
transition_recovery_control_identity(transaction, &record_name)
.source_operation_digest()
.map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
}
fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
now_unix_nanos >= transaction.not_after_unix_nanos
&& !matches!(
transaction.state,
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
)
}
async fn cleanup_terminal_transition_recovery_control(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
record_name: &str,
identity: &IlmRecoveryControlIdentity,
control_id: &str,
) -> EcstoreResult<bool> {
let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
Ok(observed) => observed,
Err(Error::ConfigNotFound) => return Ok(false),
Err(err) => return Err(err),
};
if observed.control.classification != IlmRecoveryClassification::Terminal {
return Ok(false);
}
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
let exact_source = source.is_consistent()
&& source.generation == observed.control.observed_source_generation
&& source.canonical_data.as_deref().is_some_and(|data| {
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
});
if observed.control.identity != *identity || !exact_source {
return Ok(false);
}
delete_transition_transaction_record(api, transaction).await?;
Ok(true)
}
async fn claim_transition_recovery_control(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
record_name: &str,
identity: IlmRecoveryControlIdentity,
control_id: &str,
now_unix_nanos: i64,
) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
Ok(control) => Some(control),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
if let Some(observed) = existing.as_ref() {
if observed.control.identity != identity {
return Ok(None);
}
if observed
.control
.owner
.as_ref()
.is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos)
{
let mut expired = observed.control.clone();
expired
.record_expired_attempt(now_unix_nanos)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api, observed, &expired).await?;
return Ok(None);
}
if !observed.control.should_attempt_at(now_unix_nanos) {
return Ok(None);
}
}
let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
Ok(source) => source,
Err(err) => {
if let Some(observed) = existing {
persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
return Ok(None);
}
return Err(err);
}
};
let source_matches = source.is_consistent()
&& source.canonical_data.as_deref().is_some_and(|data| {
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
});
let source_error = if source_matches {
IlmRecoveryErrorCode::None
} else if source.canonical_data.is_some() {
IlmRecoveryErrorCode::SourceGenerationChanged
} else {
IlmRecoveryErrorCode::SourceDivergent
};
let mut observed = match existing {
Some(control) => control,
None => {
let candidate = IlmRecoveryControl::new(
identity.clone(),
source.generation.clone(),
if source_matches {
IlmRecoveryClassification::Retrying
} else {
IlmRecoveryClassification::Corrupt
},
now_unix_nanos,
source_error,
)
.map_err(|err| Error::other(err.to_string()))?;
match save_recovery_control_if_absent(api.clone(), &candidate).await {
Ok(()) | Err(Error::PreconditionFailed) => {}
Err(err) => return Err(err),
}
load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
}
};
if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
return Ok(None);
}
let mut claimed = observed.control.clone();
claimed
.claim_for_source_generation(
api.id.to_string(),
Uuid::new_v4(),
now_unix_nanos,
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
source.generation,
)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
if observed.control != claimed {
return Err(Error::PreconditionFailed);
}
if !source_matches {
let mut corrupt = observed.control.clone();
corrupt
.finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api, &observed, &corrupt).await?;
return Ok(None);
}
Ok(Some(observed))
}
async fn persist_transition_recovery_source_failure(
api: Arc<ECStore>,
observed: ObservedIlmRecoveryControl,
now_unix_nanos: i64,
) -> EcstoreResult<()> {
let mut claimed = observed.control.clone();
claimed
.claim(
api.id.to_string(),
Uuid::new_v4(),
now_unix_nanos,
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
let claimed = load_recovery_control(
api.clone(),
IlmRecoveryProtocol::TransitionTransaction,
&claimed
.identity
.source_operation_digest()
.map_err(|err| Error::other(err.to_string()))?,
)
.await?;
let mut failed = claimed.control.clone();
failed
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api, &claimed, &failed).await
}
async fn refresh_transition_recovery_control_source(
api: Arc<ECStore>,
mut observed: ObservedIlmRecoveryControl,
record_name: &str,
transaction_id: Uuid,
) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
Ok(transaction) => transaction,
Err(Error::ConfigNotFound) => return Ok((observed, None)),
Err(err) => return Err(err),
};
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
let exact_source = source.is_consistent()
&& source
.canonical_data
.as_deref()
.is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
if !exact_source {
return Err(Error::PreconditionFailed);
}
if observed.control.observed_source_generation != source.generation {
let mut refreshed = observed.control.clone();
refreshed
.refresh_owned_source_generation(source.generation)
.map_err(|err| Error::other(err.to_string()))?;
save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
observed = load_recovery_control(
api,
IlmRecoveryProtocol::TransitionTransaction,
&refreshed
.identity
.source_operation_digest()
.map_err(|err| Error::other(err.to_string()))?,
)
.await?;
if observed.control != refreshed {
return Err(Error::PreconditionFailed);
}
}
Ok((observed, Some(transaction)))
}
async fn persist_transition_recovery_result(
api: Arc<ECStore>,
observed: ObservedIlmRecoveryControl,
recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
now_unix_nanos: i64,
) -> EcstoreResult<()> {
let mut next = observed.control.clone();
match recovery {
Ok(
TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
) => next
.finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
.map_err(|err| Error::other(err.to_string()))?,
Ok(TransitionTransactionRecoveryOutcome::Retained) => next
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
.map_err(|err| Error::other(err.to_string()))?,
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
.finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
.map_err(|err| Error::other(err.to_string()))?,
Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
.finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
.map_err(|err| Error::other(err.to_string()))?,
Err(err) => next
.record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
.map_err(|err| Error::other(err.to_string()))?,
}
save_recovery_control_if_current(api, &observed, &next).await
}
fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
match err {
Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
Error::ConfigNotFound
| Error::FileNotFound
| Error::FileVersionNotFound
| Error::ObjectNotFound(_, _)
| Error::VersionNotFound(_, _, _)
| Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
_ => IlmRecoveryErrorCode::Unknown,
}
}
@@ -1134,10 +1600,7 @@ async fn recover_cleanup_pending(
transaction: &TransitionTransaction,
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
match local_commit_matches_transaction(api.clone(), transaction).await {
Ok(true) => {
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
Err(err) => Err(err),
@@ -1157,7 +1620,6 @@ async fn delete_unreferenced_transition_candidate(
return Ok(TransitionTransactionRecoveryOutcome::Retained);
}
delete_transition_remote_candidate(api.clone(), &current).await?;
delete_transition_transaction_record(api, &current).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
@@ -1178,24 +1640,26 @@ async fn recover_unknown_upload_outcome(
.await
.map_err(Error::other)?
{
TransitionCandidateProbe::Missing => {
delete_transition_transaction_record(api, transaction).await?;
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
}
TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
TransitionCandidateProbe::UnversionedPresent => {
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
}
TransitionCandidateProbe::VersionedPresent(version_id)
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
{
Ok(TransitionTransactionRecoveryOutcome::Retained)
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
IlmRecoveryErrorCode::RemoteVersionUnknown,
))
}
TransitionCandidateProbe::VersionedPresent(version_id) => {
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
}
TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
Ok(TransitionTransactionRecoveryOutcome::Retained)
}
TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
IlmRecoveryErrorCode::RemoteProbeAmbiguous,
)),
TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
IlmRecoveryErrorCode::RemoteProbeUnsupported,
)),
}
}
@@ -1289,7 +1753,7 @@ pub async fn recover_transition_transaction_records(
recover_transition_transaction_records_with_now(api, limit, marker, None).await
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pub async fn recover_transition_transaction_records_at(
api: Arc<ECStore>,
limit: usize,
@@ -1323,6 +1787,11 @@ async fn recover_transition_transaction_records_with_now(
false,
)
.await?;
if list.is_truncated && list.next_continuation_token.is_none() {
return Err(Error::other(
"transition transaction recovery returned a truncated page without a continuation marker",
));
}
let mut stats = TransitionTransactionRecoveryStats {
scanned: 0,
@@ -1381,7 +1850,11 @@ async fn recover_transition_transaction_records_with_now(
) => {
stats.recovered += 1;
}
Ok(TransitionTransactionRecoveryOutcome::Retained) => {
Ok(
TransitionTransactionRecoveryOutcome::Retained
| TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
| TransitionTransactionRecoveryOutcome::OperatorRequired(_),
) => {
stats.retained += 1;
debug!(
event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
@@ -1509,11 +1982,74 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use super::*;
const BACKEND_FINGERPRINT: [u8; 32] = [7; 32];
struct RecoveryAttemptDropGuard(Arc<AtomicBool>);
impl Drop for RecoveryAttemptDropGuard {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
async fn pending_recovery_attempt(started: Arc<tokio::sync::Notify>, dropped: Arc<AtomicBool>) -> EcstoreResult<()> {
let _drop_guard = RecoveryAttemptDropGuard(dropped);
started.notify_one();
std::future::pending().await
}
#[tokio::test(start_paused = true)]
async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() {
let timeout_started = Arc::new(tokio::sync::Notify::new());
let timeout_dropped = Arc::new(AtomicBool::new(false));
let timeout_task = tokio::spawn({
let started = Arc::clone(&timeout_started);
let dropped = Arc::clone(&timeout_dropped);
async move {
await_transition_transaction_recovery(
&CancellationToken::new(),
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
pending_recovery_attempt(started, dropped),
)
.await
}
});
timeout_started.notified().await;
tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await;
let timed_out = timeout_task.await.expect("timeout wrapper task should join");
assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass");
assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt");
let cancel_token = CancellationToken::new();
let cancel_started = Arc::new(tokio::sync::Notify::new());
let cancel_dropped = Arc::new(AtomicBool::new(false));
let cancel_task = tokio::spawn({
let cancel_token = cancel_token.clone();
let started = Arc::clone(&cancel_started);
let dropped = Arc::clone(&cancel_dropped);
async move {
await_transition_transaction_recovery(
&cancel_token,
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
pending_recovery_attempt(started, dropped),
)
.await
}
});
cancel_started.notified().await;
cancel_token.cancel();
let cancelled = cancel_task.await.expect("cancellation wrapper task should join");
assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop");
assert!(
cancel_dropped.load(Ordering::SeqCst),
"outer cancellation must drop its in-flight attempt"
);
}
#[derive(Default)]
struct MemoryTransactionStore {
records: HashMap<Uuid, Vec<u8>>,
@@ -1968,5 +2504,19 @@ mod tests {
transition_transaction_record_object_name(Uuid::nil()),
Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
));
assert_eq!(
transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
transaction_id
);
for malformed in [
object.to_ascii_uppercase(),
object.replace("/aa/aa/", "/ff/aa/"),
object.replace("/aa/aa/", "/aa/aa/extra/"),
] {
assert!(matches!(
transition_transaction_id_from_record_object_name(&malformed),
Err(TransitionTransactionError::Corrupt(_))
));
}
}
}
+28
View File
@@ -1611,6 +1611,34 @@ mod test {
assert!(bm.bucket_target_config.is_none());
}
/// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
/// already carries as a compatibility fixture stores
/// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
/// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
/// bytes below are the exact payload the fixture in
/// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
/// ever grows the array-shaped compatibility parse, this test is where the
/// upgrade break is pinned and where the decision has to be recorded.
#[test]
fn minio_array_shaped_bucket_targets_are_unreadable() {
let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
let mut bm = BucketMetadata::new("minio-array-targets");
bm.bucket_targets_config_json = minio_array.clone();
bm.parse_all_configs()
.expect("a MinIO-shaped targets blob must not fail the whole metadata load");
assert!(
bm.bucket_targets_unreadable(),
"an array-shaped MinIO targets blob is unreadable, not an empty target set"
);
assert!(bm.bucket_target_config.is_none());
assert_eq!(
bm.bucket_targets_config_json, minio_array,
"the raw MinIO bytes must survive so the configuration stays recoverable"
);
}
/// The invariant every branch of `parse_all_configs` shares: a stored but
/// undecodable payload keeps its raw bytes and leaves the typed field
/// `None`, so no branch fabricates a value. What a reader may then do with
+26 -5
View File
@@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
let id_kind = parts
.next()
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id"))
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
let source_path = parts
.next()
@@ -3023,8 +3023,9 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
}
match id_kind {
"operation_id" if !is_sha256_checksum(id) => {
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id")));
"operation_id" | "control_id" if !is_sha256_checksum(id) => {
let id_label = id_kind.trim_end_matches("_id");
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
}
"transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
@@ -19300,8 +19301,8 @@ mod pools_tests {
load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica,
reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
@@ -20169,6 +20170,26 @@ mod pools_tests {
assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
}
#[test]
fn decommission_recovery_control_receipt_path_round_trips() {
let run_token = "b".repeat(64);
let control_id = "a".repeat(64);
let source_path = format!(
"ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
&control_id[..2],
&control_id[2..4],
control_id
);
let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
assert_eq!(locator.run_token, run_token);
assert_eq!(locator.source_path, source_path);
assert_eq!(locator.id_kind, "control_id");
assert_eq!(locator.id, control_id);
}
#[test]
fn decommission_receipt_merge_preserves_terminal_proof() {
let operation_id = "a".repeat(64);
+6
View File
@@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
+62 -7
View File
@@ -324,6 +324,30 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
}
impl LocalDiskWrapper {
pub(in crate::disk) async fn undo_write_with_namespace_owner(
&self,
volume: &str,
path: &str,
fi: FileInfo,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
self.track_disk_health_mutation(
"delete_version",
DiskMetricMutation::Delete,
|| async {
// Preserve the old DiskAPI future's boxing boundary.
Box::pin(
self.disk
.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner),
)
.await
},
get_max_timeout_duration(),
)
.await
}
pub(in crate::disk) async fn rename_data_observed(
&self,
src_volume: &str,
@@ -333,6 +357,34 @@ impl LocalDiskWrapper {
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> super::RenameDataObservation {
self.rename_data_observed_with_guards(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
super::RenameDataGuards {
external_guard,
..Default::default()
},
)
.await
}
pub(in crate::disk) async fn rename_data_observed_with_guards(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
guards: super::RenameDataGuards,
) -> super::RenameDataObservation {
let super::RenameDataGuards {
external_guard,
namespace_owner,
..
} = guards;
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -357,13 +409,15 @@ impl LocalDiskWrapper {
DiskMetricMutation::Write,
|| async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed =
Box::pin(
operation
.disk
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
)
.await;
let observed = Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
))
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
@@ -1301,6 +1355,7 @@ impl LocalDiskWrapper {
self.disk.get_object_path(volume, path)
}
#[cfg(unix)]
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
self.disk.get_object_path_for_io(volume, path)
}
+2
View File
@@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
fs::rename(from, to).await
}
#[cfg(any(not(windows), test))]
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
std::fs::rename(from, to)
}
#[cfg(any(not(windows), test))]
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
File diff suppressed because it is too large Load Diff
+44 -14
View File
@@ -17,13 +17,14 @@
#[cfg(all(test, windows))]
use super::run_destination_commit_directory_preparation;
#[cfg(any(not(windows), test))]
use super::should_fail_local_inline_rollback_hardlink;
use super::{
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
skip_access_checks,
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
};
#[cfg(test)]
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
@@ -33,7 +34,7 @@ use crate::disk::{
error::{DiskError, Result},
error_conv::{to_access_error, to_file_error},
os,
os::{check_path_length, rename_all},
os::check_path_length,
};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileMeta};
@@ -73,6 +74,8 @@ fn rollback_inline_metadata_commit_std(
rollback_data_dir: Option<Uuid>,
local_rollback_path: Option<&Path>,
) -> std::io::Result<()> {
#[cfg(all(test, not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
if let Some(backup_path) = local_rollback_path {
// The commit immediately before this rollback renamed the staged
// xl.meta from the same directory as `backup_path` onto
@@ -86,6 +89,7 @@ fn rollback_inline_metadata_commit_std(
Ok(())
}
#[cfg(any(not(windows), test))]
pub(super) fn create_local_inline_rollback_backup(
dst_file_path: &Path,
staging_file_path: &Path,
@@ -231,6 +235,12 @@ async fn restore_published_data_source(
#[derive(Debug)]
pub(in crate::disk) struct LocalRenamePreflightRejection(());
#[derive(Default)]
pub(super) struct RenameDataState {
namespace_owner: Option<Arc<dyn Send + Sync>>,
preflight_rejection: Option<LocalRenamePreflightRejection>,
}
impl LocalDisk {
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
pub(super) async fn rename_data_inner(
@@ -240,7 +250,7 @@ impl LocalDisk {
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
preflight_rejection: &mut Option<LocalRenamePreflightRejection>,
state: &mut RenameDataState,
) -> Result<RenameDataResp> {
crate::hp_guard!("LocalDisk::rename_data");
let mut fi = fi;
@@ -269,7 +279,13 @@ impl LocalDisk {
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
None => None,
};
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
&self.root,
dst_volume,
&destination_object_path,
state.namespace_owner.take(),
)
.await;
if let Some(claim) = quota_fence_claim {
mutation_lease.attach_external_guard(claim);
}
@@ -302,7 +318,7 @@ impl LocalDisk {
error = %e,
"Disk local access check failed"
);
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -320,7 +336,7 @@ impl LocalDisk {
error = %e,
"Disk local access check failed"
);
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
@@ -528,7 +544,9 @@ impl LocalDisk {
// rename below.
if fi_healing
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
&& let Err(err) = self.move_to_trash(dst_data_path, true, false).await
&& let Err(err) = self
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
.await
{
warn!(
target: "rustfs_ecstore::disk::local",
@@ -755,7 +773,7 @@ impl LocalDisk {
&& let Some(parent) = dst_file_path.parent()
{
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) = os::fsync_dst_dir_group_commit(parent).await {
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
fsync_started,
@@ -793,7 +811,7 @@ impl LocalDisk {
break;
}
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) = os::fsync_dir(dir).await {
if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
fsync_started,
@@ -1024,7 +1042,15 @@ impl LocalDisk {
// rename_all acquires the backup path's namespace lease. Do not
// hold a disk admission while acquiring another namespace lock.
drop(file_sync_admission.take());
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
if let Err(err) = os::rename_all_with_owner(
staged_backup,
&backup_path,
&dst_volume_dir,
&self.publication_root,
Some(mutation_lease.clone()),
)
.await
{
let _ = remove_file_if_exists(staged_backup);
return Err(err);
}
@@ -1220,14 +1246,18 @@ impl LocalDisk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> super::super::RenameDataObservation {
let mut preflight_rejection = None;
let mut state = RenameDataState {
namespace_owner,
..Default::default()
};
let result = self
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection)
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state)
.await;
super::super::RenameDataObservation {
result,
preflight_rejection,
preflight_rejection: state.preflight_rejection,
}
}
}
+37 -3
View File
@@ -75,6 +75,14 @@ use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
/// Independent admission and physical ownership for one disk rename.
#[derive(Default)]
pub(crate) struct RenameDataGuards {
pub(crate) scanner_publication_lease_token: Option<Uuid>,
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
}
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
pub(crate) struct RenameDataObservation {
pub(crate) result: Result<RenameDataResp>,
@@ -190,11 +198,17 @@ pub struct MmapCopyStageMetrics {
pub(crate) path_resolve_stage: &'static str,
pub(crate) metadata_lookup_stage: &'static str,
pub(crate) metadata_validate_stage: &'static str,
#[cfg(unix)]
pub(crate) blocking_wait_stage: &'static str,
#[cfg(unix)]
pub(crate) blocking_task_stage: &'static str,
#[cfg(unix)]
pub(crate) file_open_stage: &'static str,
#[cfg(unix)]
pub(crate) mmap_map_stage: &'static str,
#[cfg(unix)]
pub(crate) mmap_copy_stage: &'static str,
#[cfg(unix)]
pub(crate) direct_read_copy_stage: &'static str,
}
@@ -718,6 +732,25 @@ impl Disk {
}
}
/// Keep local undo publication owned independently of the wrapper deadline.
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
pub(crate) async fn undo_write_with_namespace_owner(
&self,
volume: &str,
path: &str,
fi: FileInfo,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
match self {
Self::Local(disk) => {
disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner)
.await
}
Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await,
}
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -737,12 +770,12 @@ impl Disk {
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
guards: RenameDataGuards,
) -> RenameDataObservation {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
@@ -753,7 +786,7 @@ impl Disk {
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
guards.scanner_publication_lease_token,
)
.await,
),
@@ -922,6 +955,7 @@ impl Disk {
}
}
#[cfg(unix)]
pub(crate) fn get_object_path_for_io_if_local(
&self,
volume: &str,
+524 -51
View File
@@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder {
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
#[cfg(unix)]
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
@@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder {
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
}
#[cfg(unix)]
pub(crate) fn record_limited(dir: &Path) {
record_path(&LIMITED, dir, "limited fsync dir recorder");
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
@@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder {
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
}
#[cfg(unix)]
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
BEFORE_LIMITED
.lock()
@@ -237,11 +240,56 @@ pub(crate) mod fsync_dir_recorder {
.insert(dir.to_path_buf(), kind);
}
#[cfg(unix)]
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
}
}
/// Pause a real namespace mutation inside its physical executor.
#[cfg(all(test, not(windows)))]
pub(crate) mod prepared_publication_test_hooks {
use super::*;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub(crate) enum Stage {
PreparedRename,
Rename,
Remove,
Rollback,
DirFsync,
}
type Hook = Box<dyn FnOnce() + Send>;
type Key = (Stage, PathBuf);
static BEFORE_PUBLICATION: LazyLock<Mutex<HashMap<Key, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
pub(crate) struct Guard(Key);
impl Drop for Guard {
fn drop(&mut self) {
BEFORE_PUBLICATION.lock().remove(&self.0);
}
}
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
install_at(Stage::PreparedRename, path, hook)
}
pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
let key = (stage, path.to_path_buf());
assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none());
Guard(key)
}
pub(crate) fn run(stage: Stage, path: &Path) {
let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf()));
if let Some(hook) = hook {
hook();
}
}
}
#[cfg(all(test, windows))]
pub(crate) mod windows_rename_test_hooks {
use super::*;
@@ -576,6 +624,7 @@ impl OpenedDstDirFsyncGroup {
}
struct DstDirFsyncWaiter {
namespace_owner: Option<Arc<dyn Send + Sync>>,
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
}
@@ -634,6 +683,7 @@ impl DstDirFsyncGroupCommit {
fn enqueue_opened(
&self,
opened: OpenedDstDirFsyncGroup,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
let (result_tx, result_rx) = oneshot::channel();
let mut registry = self.inner.lock();
@@ -664,7 +714,10 @@ impl DstDirFsyncGroupCommit {
group
};
let mut group_state = group.inner.lock();
group_state.pending.push_back(DstDirFsyncWaiter { result_tx });
group_state.pending.push_back(DstDirFsyncWaiter {
result_tx,
namespace_owner,
});
let start_worker = !group_state.worker_running;
if start_worker {
group_state.worker_running = true;
@@ -686,7 +739,13 @@ impl DstDirFsyncGroupCommit {
fn remove_idle_group(&self, group: &Arc<DstDirFsyncGroup>) {
let mut registry = self.inner.lock();
let group_state = group.inner.lock();
if !group_state.worker_running && group_state.pending.is_empty() {
if !group_state.worker_running
&& group_state.pending.is_empty()
&& registry
.groups
.get(&group.key)
.is_some_and(|registered| Arc::ptr_eq(registered, group))
{
registry.groups.remove(&group.key);
}
}
@@ -709,16 +768,20 @@ impl DstDirFsyncGroupCommit {
&self,
dir: &Path,
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?)
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None)
}
}
#[cfg(unix)]
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
#[cfg(test)]
let dir = group.dir.clone();
let dir_file = group.dir_file.clone();
fsync_spawn_blocking(move || {
// The batch worker may be cancelled while this syscall is still running.
let _namespace_owners = namespace_owners;
#[cfg(all(test, not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
#[cfg(test)]
{
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -733,66 +796,118 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
}
#[cfg(not(unix))]
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
let _namespace_owners = namespace_owners;
fsync_dir(&group.dir).await
}
async fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) {
loop {
#[cfg(test)]
fsync_dir_recorder::run_before_group_batch(&group.dir);
tokio::task::yield_now().await;
let batch: Vec<DstDirFsyncWaiter> = {
let mut group_state = group.inner.lock();
group_state.pending.drain(..).collect()
};
if batch.is_empty() {
let mut group_state = group.inner.lock();
struct DstDirFsyncWorkerGuard {
group: Arc<DstDirFsyncGroup>,
in_flight: usize,
armed: bool,
}
impl Drop for DstDirFsyncWorkerGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
// Cancellation must release queued owners, but the physical batch keeps
// its own owners until its blocking syscall returns.
let pending = {
let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock();
let mut group_state = self.group.inner.lock();
let pending = std::mem::take(&mut group_state.pending);
group_state.worker_running = false;
drop(group_state);
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
return;
}
#[cfg(test)]
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
let result = fsync_open_dst_dir_group(&group)
.await
.map_err(SharedDstDirFsyncError::from_error);
let batch_len = batch.len();
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
let should_stop = {
let mut group_state = group.inner.lock();
if group_state.pending.is_empty() {
group_state.worker_running = false;
true
} else {
false
if registry
.groups
.get(&self.group.key)
.is_some_and(|group| Arc::ptr_eq(group, &self.group))
{
registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight);
registry.groups.remove(&self.group.key);
}
pending
};
if should_stop {
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
}
for waiter in batch {
let _ = waiter.result_tx.send(result.clone());
}
if should_stop {
return;
// Lease and channel destructors must run outside the registry locks.
drop(pending);
}
}
fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) -> impl std::future::Future<Output = ()> {
// Capture before spawning: shutdown may drop the future without polling it.
let worker_guard = DstDirFsyncWorkerGuard {
group: group.clone(),
in_flight: 0,
armed: true,
};
async move {
let mut worker_guard = worker_guard;
loop {
#[cfg(test)]
fsync_dir_recorder::run_before_group_batch(&group.dir);
tokio::task::yield_now().await;
let mut batch: Vec<DstDirFsyncWaiter> = {
let mut group_state = group.inner.lock();
group_state.pending.drain(..).collect()
};
if batch.is_empty() {
let mut group_state = group.inner.lock();
worker_guard.armed = false;
group_state.worker_running = false;
drop(group_state);
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
return;
}
worker_guard.in_flight = batch.len();
#[cfg(test)]
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect();
let result = fsync_open_dst_dir_group(&group, namespace_owners)
.await
.map_err(SharedDstDirFsyncError::from_error);
let batch_len = batch.len();
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
worker_guard.in_flight = 0;
let should_stop = {
let mut group_state = group.inner.lock();
if group_state.pending.is_empty() {
worker_guard.armed = false;
group_state.worker_running = false;
true
} else {
false
}
};
if should_stop {
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
}
for waiter in batch {
let _ = waiter.result_tx.send(result.clone());
}
if should_stop {
return;
}
}
}
}
async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
async fn fsync_dst_dir_group_commit_with_enabled(
dir: impl AsRef<Path>,
enabled: bool,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
if !enabled {
return fsync_dir(dir).await;
return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await;
}
let dir = dir.as_ref().to_path_buf();
let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir))
.await
.map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??;
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?;
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?;
if let Some(group) = worker {
tokio::spawn(run_dst_dir_fsync_group_worker(group));
}
@@ -804,8 +919,11 @@ async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled:
}
}
pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef<Path>) -> io::Result<()> {
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await
pub(crate) async fn fsync_dst_dir_group_commit(
dir: impl AsRef<Path>,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await
}
pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
@@ -814,7 +932,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
admission: &FileSyncAdmission,
) -> io::Result<()> {
if dst_dir_fsync_group_commit_enabled() {
fsync_dst_dir_group_commit_with_enabled(dir, true).await
fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await
} else {
fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await
}
@@ -822,7 +940,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
#[cfg(test)]
pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
fsync_dst_dir_group_commit_with_enabled(dir, enabled).await
fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await
}
#[cfg(test)]
@@ -1229,6 +1347,8 @@ pub(crate) struct NamespaceMutationLease {
_namespace_guard: OwnedMutexGuard<()>,
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
// Independent of the quota claim; both survive cancellation of the waiter.
_namespace_owner: Option<Arc<dyn Send + Sync>>,
}
impl NamespaceMutationLease {
@@ -1238,10 +1358,18 @@ impl NamespaceMutationLease {
}
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
acquire_namespace_mutation_lease_with_owner(path, None).await
}
async fn acquire_namespace_mutation_lease_with_owner(
path: &Path,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Arc<NamespaceMutationLease> {
Arc::new(NamespaceMutationLease {
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
_volume_guard: None,
external_guard: Mutex::new(None),
_namespace_owner: namespace_owner,
})
}
@@ -1251,6 +1379,15 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
root: &Path,
volume: &str,
destination_object: &Path,
) -> Arc<NamespaceMutationLease> {
acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await
}
pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
root: &Path,
volume: &str,
destination_object: &Path,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Arc<NamespaceMutationLease> {
let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await;
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
@@ -1258,6 +1395,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
_namespace_guard: namespace_guard,
_volume_guard: Some(volume_guard),
external_guard: Mutex::new(None),
_namespace_owner: namespace_owner,
})
}
@@ -1747,6 +1885,69 @@ pub async fn rename_all(
Ok(())
}
pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
#[cfg(unix)]
{
if namespace_owner.is_none() {
return fsync_dir(path).await;
}
let path = path.to_path_buf();
fsync_spawn_blocking(move || {
let _namespace_owner = namespace_owner;
fsync_dir_std(path)
})
.await?
}
#[cfg(not(unix))]
{
let _ = namespace_owner;
fsync_dir(path).await
}
}
/// Retain namespace ownership in the actual filesystem executor after timeout.
pub(crate) async fn remove_file_with_owner(
path: impl AsRef<Path>,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
if namespace_owner.is_none() {
return tokio::fs::remove_file(path).await;
}
let path = path.as_ref().to_path_buf();
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
run_blocking_namespace_operation(lease, move || {
#[cfg(all(test, not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
std::fs::remove_file(path)
})
.await
}
/// Retain namespace ownership in the actual filesystem executor after timeout.
pub(crate) async fn remove_dir_with_owner(
path: impl AsRef<Path>,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
if namespace_owner.is_none() {
return tokio::fs::remove_dir(path).await;
}
let path = path.as_ref().to_path_buf();
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
}
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
pub(crate) async fn rename_all_with_owner(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
publication_root: &PublicationRoot,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
}
pub(crate) async fn rename_all_with_lease(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
@@ -1939,6 +2140,8 @@ pub(crate) async fn rename_all_with_prepared_source(
move || {
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
#[cfg(test)]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
rename_prepared(&src_file_path, &dst_file_path, &preparation)
}
};
@@ -1977,6 +2180,32 @@ pub async fn rename_all_ignore_missing_source(
}
}
#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)]
pub(crate) async fn rename_all_ignore_missing_source_with_owner(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
publication_root: &PublicationRoot,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
let src_file_path = src_file_path.as_ref();
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
match reliable_rename_inner_with_lease(
src_file_path.to_path_buf(),
dst_file_path.as_ref().to_path_buf(),
base_dir.as_ref().to_path_buf(),
publication_root.clone(),
false,
lease,
)
.await
{
Ok(()) => Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()),
Err(err) => Err(to_file_error(err).into()),
}
}
#[cfg(windows)]
pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool {
let Some(source_parent) = src_file_path.parent() else {
@@ -2042,6 +2271,11 @@ async fn reliable_rename_inner_with_lease(
let base_dir = base_dir.clone();
move || {
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
#[cfg(all(test, not(windows)))]
{
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
}
rename_prepared(&src_file_path, &dst_file_path, &preparation)
}
};
@@ -6136,6 +6370,245 @@ mod tests {
wait_for_dst_dir_fsync_group_commit_idle().await;
}
#[cfg(unix)]
#[tokio::test]
#[serial_test::serial(dst_dir_fsync_group_commit)]
async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() {
let temp_dir = tempdir().expect("fixture directory");
let dir = temp_dir.path().canonicalize().expect("canonical fsync path");
let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let first_owner = first_ctx.begin_namespace_commit();
let second_owner = second_ctx.begin_namespace_commit();
let first_probe = Arc::downgrade(&first_owner);
let second_probe = Arc::downgrade(&second_owner);
let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_opened(
OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"),
Some(first_owner),
)
.expect("queue first real waiter");
let group = group.expect("first waiter starts the group");
let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_opened(
OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"),
Some(second_owner),
)
.expect("queue second real waiter");
assert!(second_worker.is_none(), "same directory must join the same batch");
assert_eq!(group.inner.lock().pending.len(), 2);
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook =
prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone()));
tokio::time::timeout(Duration::from_secs(5), entered_rx)
.await
.expect("batch must reach its physical fsync")
.expect("physical fsync entry");
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]);
assert!(
group.inner.lock().pending.is_empty(),
"both waiters were transferred into the physical batch"
);
let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let queued_owner = queued_ctx.begin_namespace_commit();
let queued_probe = Arc::downgrade(&queued_owner);
let queued_generation = queued_ctx.namespace_commit_generation();
let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_opened(
OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"),
Some(queued_owner),
)
.expect("queue a waiter after the physical batch was frozen");
assert!(queued_worker.is_none());
assert_eq!(group.inner.lock().pending.len(), 1);
drop((first_rx, second_rx));
worker.abort();
assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled());
assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation");
assert!(queued_probe.upgrade().is_none());
assert!(!queued_ctx.namespace_commits_pending());
assert!(queued_ctx.namespace_commit_generation() > queued_generation);
assert!(group.inner.lock().pending.is_empty());
assert!(!group.inner.lock().worker_running);
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some();
let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some();
let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation());
drop(release_tx);
tokio::time::timeout(Duration::from_secs(5), async {
while Arc::strong_count(&group.dir_file) != 1
|| first_probe.upgrade().is_some()
|| second_probe.upgrade().is_some()
|| first_ctx.namespace_commits_pending()
|| second_ctx.namespace_commits_pending()
{
tokio::task::yield_now().await;
}
})
.await
.expect("physical fsync must release every batch owner");
assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute");
assert!(
first_pending && second_pending,
"one physical batch must preserve both independent namespace owners"
);
assert!(!first_ctx.namespace_commits_pending());
assert!(!second_ctx.namespace_commits_pending());
assert!(first_ctx.namespace_commit_generation() > generations.0);
assert!(second_ctx.namespace_commit_generation() > generations.1);
}
#[cfg(unix)]
#[tokio::test]
#[serial_test::serial(dst_dir_fsync_group_commit)]
async fn grouped_fsync_unpolled_worker_releases_queued_owner() {
let temp_dir = tempdir().expect("fixture directory");
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let owner = ctx.begin_namespace_commit();
let probe = Arc::downgrade(&owner);
let generation = ctx.namespace_commit_generation();
let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_opened(
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"),
Some(owner),
)
.expect("queue a real waiter");
let group = group.expect("first waiter starts the group");
let worker = run_dst_dir_fsync_group_worker(group.clone());
assert!(ctx.namespace_commits_pending());
drop(worker);
assert!(rx.await.is_err(), "shutdown before first poll must release the waiter");
assert!(probe.upgrade().is_none());
assert!(!ctx.namespace_commits_pending());
assert!(ctx.namespace_commit_generation() > generation);
assert!(group.inner.lock().pending.is_empty());
assert!(!group.inner.lock().worker_running);
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
assert!(
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(),
"the dropped future must not dispatch a physical batch"
);
}
#[cfg(unix)]
#[test]
fn stale_idle_group_cleanup_preserves_successor_registration() {
let temp_dir = tempdir().expect("fixture directory");
let registry = DstDirFsyncGroupCommit::default();
let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker");
let old_group = first_worker.expect("first waiter starts a worker");
// W1 has completed its batch and marked G idle, but has not cleaned G up.
let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter");
registry.complete_batch(1);
old_group.inner.lock().worker_running = false;
let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker");
let reused_group = second_worker.expect("idle G starts another worker");
assert!(Arc::ptr_eq(&old_group, &reused_group));
let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter");
registry.complete_batch(1);
reused_group.inner.lock().worker_running = false;
registry.remove_idle_group(&reused_group);
assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G");
assert!(second_waiter.result_tx.send(Ok(())).is_ok());
assert!(second_rx.try_recv().expect("second worker reports completion").is_ok());
let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor");
let successor = successor_worker.expect("successor starts a new group");
assert!(!Arc::ptr_eq(&old_group, &successor));
assert_eq!(registry.counts_for_test(), (1, 1));
// W1 resumes with its old Arc after W2 removed G and W3 installed G2.
registry.remove_idle_group(&old_group);
assert!(first_waiter.result_tx.send(Ok(())).is_ok());
assert!(first_rx.try_recv().expect("first worker reports completion").is_ok());
assert!(
registry
.inner
.lock()
.groups
.get(&successor.key)
.is_some_and(|registered| Arc::ptr_eq(registered, &successor)),
"stale cleanup must retain the exact successor Arc"
);
assert_eq!(registry.counts_for_test(), (1, 1));
assert!(successor.inner.lock().worker_running);
assert_eq!(successor.inner.lock().pending.len(), 1);
assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)));
let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor");
assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3");
assert_eq!(successor.inner.lock().pending.len(), 2);
assert_eq!(registry.counts_for_test(), (1, 2));
}
#[cfg(unix)]
#[tokio::test]
#[serial_test::serial(dst_dir_fsync_group_commit)]
async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() {
wait_for_dst_dir_fsync_group_commit_idle().await;
let temp_dir = tempdir().expect("fixture directory");
let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_for_test(temp_dir.path())
.expect("enqueue old group");
let old_group = old_worker.expect("old group starts a worker");
tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone()))
.await
.expect("old worker must finish its actual fsync");
assert!(old_rx.await.expect("old worker reports completion").is_ok());
assert!(fsync_dir_recorder::was_fsynced(temp_dir.path()));
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]);
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let owner = ctx.begin_namespace_commit();
let probe = Arc::downgrade(&owner);
let generation = ctx.namespace_commit_generation();
let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT
.enqueue_opened(
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"),
Some(owner),
)
.expect("enqueue successor owner");
let successor = successor_worker.expect("successor starts a new group");
assert!(!Arc::ptr_eq(&old_group, &successor));
let worker = run_dst_dir_fsync_group_worker(successor.clone());
// The stale Arc represents W1 resuming after another worker removed G.
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group);
assert!(ctx.namespace_commits_pending());
assert!(probe.upgrade().is_some());
drop(worker);
let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx)
.await
.expect("dropping the unpolled worker must release its channel")
.is_err();
let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test();
let owner_released = probe.upgrade().is_none();
let namespace_pending = ctx.namespace_commits_pending();
let generation_after_drop = ctx.namespace_commit_generation();
let successor_pending = successor.inner.lock().pending.len();
let worker_running = successor.inner.lock().worker_running;
// Preserve the observed result before cleanup, so a RED run cannot leak
// its phantom count into unrelated tests in the same process.
clear_dst_dir_fsync_group_commit_for_test();
assert!(channel_closed);
assert!(owner_released);
assert!(!namespace_pending);
assert!(generation_after_drop > generation);
assert_eq!(successor_pending, 0);
assert!(!worker_running);
assert_eq!(
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()),
vec![1],
"dropping the successor before its first poll must not dispatch another fsync"
);
assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(dst_dir_fsync_group_commit)]
async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() {
+12 -3
View File
@@ -13,12 +13,15 @@
// limitations under the License.
use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
};
#[cfg(unix)]
use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
};
#[cfg(feature = "hotpath")]
use crate::disk::FileWriter;
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
@@ -406,11 +409,17 @@ async fn open_disk_reader(
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
#[cfg(unix)]
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
#[cfg(unix)]
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
#[cfg(unix)]
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
#[cfg(unix)]
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
#[cfg(unix)]
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
#[cfg(unix)]
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
});
let mmap_result = {
@@ -56,6 +56,7 @@
use std::collections::HashMap;
use std::io::Cursor;
#[cfg(feature = "test-util")]
use std::path::Path;
use std::sync::{
Arc,
@@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt;
use tokio::sync::{Mutex, Notify, RwLock};
use uuid::Uuid;
#[cfg(feature = "test-util")]
use crate::disk::endpoint::Endpoint;
#[cfg(feature = "test-util")]
use crate::disk::format::FormatV3;
#[cfg(feature = "test-util")]
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
use crate::services::tier::tier::TierConfigMgr;
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
use crate::services::tier::warm_backend::{
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
#[cfg(feature = "test-util")]
use rustfs_filemeta::FileMeta;
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
#[cfg(feature = "test-util")]
use rustfs_utils::path::path_join_buf;
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
#[cfg(feature = "test-util")]
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
#[cfg(feature = "test-util")]
impl TransitionCleanupStoreBarrier {
/// Install the barrier for the next rejected transition cleanup.
pub fn install() -> Self {
@@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier {
}
/// Default polling cadence used by the `wait_for_*` helpers.
#[cfg(feature = "test-util")]
const POLL_INTERVAL: Duration = Duration::from_millis(50);
/// A fault to inject into [`MockWarmBackend`] operations.
@@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard {
}
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
#[cfg(feature = "test-util")]
pub struct MockPutBarrier {
state: Arc<MockPutBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockPutBarrier {
/// Wait until the remote body is stored and the PUT is paused before returning.
pub async fn wait_until_paused(&self) {
@@ -226,6 +237,7 @@ impl MockPutBarrier {
}
}
#[cfg(feature = "test-util")]
impl Drop for MockPutBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -258,10 +270,12 @@ impl Drop for MockGetBarrier {
}
/// One-shot barrier that pauses and then fails a mock tier DELETE.
#[cfg(feature = "test-util")]
pub struct MockRemoveBarrier {
state: Arc<MockRemoveBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockRemoveBarrier {
/// Wait until DELETE reaches the deterministic failure point.
pub async fn wait_until_paused(&self) {
@@ -283,6 +297,7 @@ impl MockRemoveBarrier {
}
}
#[cfg(feature = "test-util")]
impl Drop for MockRemoveBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -306,6 +321,7 @@ impl MockWarmBackend {
}
/// Arm a one-shot pause after the next tier PUT stores its remote body.
#[cfg(feature = "test-util")]
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
let state = Arc::new(MockPutBarrierState::default());
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
@@ -313,6 +329,7 @@ impl MockWarmBackend {
}
/// Pause and then fail the next DELETE after it reaches the backend.
#[cfg(feature = "test-util")]
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
let state = Arc::new(MockRemoveBarrierState::default());
let mut barrier = self.inner.remove_barrier.lock().await;
@@ -323,6 +340,7 @@ impl MockWarmBackend {
/// Arm a one-shot pause before the next tier GET, then return an error
/// after the test releases it.
#[cfg(feature = "test-util")]
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
let state = Arc::new(MockGetBarrierState {
fail_after_release: true,
@@ -343,6 +361,7 @@ impl MockWarmBackend {
// ---- fault injection -------------------------------------------------
/// Replace the entire fault configuration.
#[cfg(feature = "test-util")]
pub async fn set_faults(&self, faults: FaultConfig) {
*self.inner.faults.lock().await = faults;
}
@@ -353,6 +372,7 @@ impl MockWarmBackend {
}
/// Toggle "HTTP 5xx" server errors on every operation.
#[cfg(feature = "test-util")]
pub async fn set_server_error(&self, server_error: bool) {
self.inner.faults.lock().await.server_error = server_error;
}
@@ -363,11 +383,13 @@ impl MockWarmBackend {
}
/// Set (or clear, with `None`) injected latency applied before each op.
#[cfg(feature = "test-util")]
pub async fn set_latency(&self, latency: Option<Duration>) {
self.inner.faults.lock().await.latency = latency;
}
/// Clear all injected faults, restoring healthy behaviour.
#[cfg(feature = "test-util")]
pub async fn clear_faults(&self) {
*self.inner.faults.lock().await = FaultConfig::default();
}
@@ -375,6 +397,7 @@ impl MockWarmBackend {
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
/// the complete body. This models a backend that incorrectly accepts a
/// truncated stream while still returning success.
#[cfg(feature = "test-util")]
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
*self.inner.put_read_limit.lock().await = limit;
}
@@ -395,12 +418,14 @@ impl MockWarmBackend {
}
/// Reject non-empty remote versions before transition metadata is committed.
#[cfg(feature = "test-util")]
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
}
/// Reject the next non-empty remote version validation without changing
/// subsequent exact-version backend cleanup behavior.
#[cfg(feature = "test-util")]
pub fn reject_next_non_empty_remote_version_validation(&self) {
self.inner
.reject_non_empty_remote_version_validations
@@ -438,6 +463,7 @@ impl MockWarmBackend {
}
/// Clear the operation log without touching stored objects or faults.
#[cfg(feature = "test-util")]
pub async fn clear_op_log(&self) {
self.inner.op_log.lock().await.clear();
}
@@ -459,11 +485,13 @@ impl MockWarmBackend {
}
/// Return the exact object/version pairs produced by successful tier PUTs.
#[cfg(feature = "test-util")]
pub async fn put_versions(&self) -> Vec<(String, String)> {
self.inner.put_versions.lock().await.clone()
}
/// Return the exact object/version pairs passed to successful tier removes.
#[cfg(feature = "test-util")]
pub async fn remove_versions(&self) -> Vec<(String, String)> {
self.inner.remove_versions.lock().await.clone()
}
@@ -475,6 +503,7 @@ impl MockWarmBackend {
/// Number of `get` calls recorded — useful to assert restore reads hit the
/// local copy rather than the remote tier.
#[cfg(feature = "test-util")]
pub async fn get_count(&self) -> usize {
self.inner
.op_log
@@ -486,6 +515,7 @@ impl MockWarmBackend {
}
/// Number of `put` calls recorded.
#[cfg(feature = "test-util")]
pub async fn put_count(&self) -> usize {
self.inner
.op_log
@@ -499,6 +529,7 @@ impl MockWarmBackend {
// ---- storage inspection ---------------------------------------------
/// Whether the backend currently stores `object`.
#[cfg(feature = "test-util")]
pub async fn contains(&self, object: &str) -> bool {
self.inner.objects.lock().await.contains_key(object)
}
@@ -509,11 +540,13 @@ impl MockWarmBackend {
}
/// A clone of the stored object, if present.
#[cfg(feature = "test-util")]
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
self.inner.objects.lock().await.get(object).cloned()
}
/// A clone of the raw bytes stored for `object`, if present.
#[cfg(feature = "test-util")]
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
}
@@ -538,6 +571,7 @@ impl MockWarmBackend {
/// Poll until `object` is absent from the backend, or `timeout` elapses.
/// Returns `true` if the object disappeared within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -553,6 +587,7 @@ impl MockWarmBackend {
/// Poll until the backend holds exactly `expected` objects, or `timeout`
/// elapses. Returns `true` if the count was reached within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
/// free-version count.
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg(feature = "test-util")]
pub struct TransitionMeta {
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
pub status: String,
@@ -860,6 +896,7 @@ pub struct TransitionMeta {
pub free_version_count: usize,
}
#[cfg(feature = "test-util")]
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
// with the position recorded in the disk's own format.json, so derive the
@@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
/// The free-version metadata removal lands asynchronously after the remote
/// object disappears, so callers typically poll via
/// [`wait_for_free_version_absence`] instead of asserting a single read.
#[cfg(feature = "test-util")]
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
let Some(disk) = open_disk(disk_path).await else {
return 0;
@@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
/// fields are taken from the newest version that carries a transition record;
/// if no version is transitioned, they are taken from the current version (and
/// will be empty).
#[cfg(feature = "test-util")]
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
let disk = open_disk(disk_path).await?;
let data = disk
@@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
/// disk is missing the object or disagrees — this is the shard-consistency
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
/// four-tuple plus free-version count must match across all erasure shards).
#[cfg(feature = "test-util")]
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
@@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
/// elapses. Returns `true` if the free versions drained within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -1040,6 +1081,44 @@ mod tests {
);
}
#[tokio::test]
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
let backend = MockWarmBackend::new();
let metadata = HashMap::from([
("content-type".to_string(), "text/plain".to_string()),
("project".to_string(), "archive".to_string()),
]);
let version = backend
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
.await
.expect("mock PUT should preserve remote metadata");
assert_eq!(backend.metadata("object").await, Some(metadata));
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe stored object"),
TransitionCandidateProbe::VersionedPresent(version)
);
backend.external_remove("object").await;
assert_eq!(backend.metadata("object").await, None);
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe removed object"),
TransitionCandidateProbe::Missing
);
let operations = backend.op_log().await;
assert!(
operations
.iter()
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
);
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
}
#[tokio::test]
async fn mock_probe_preserves_fault_fail_closed_behavior() {
let backend = MockWarmBackend::new();
+470 -57
View File
@@ -143,11 +143,12 @@ struct TierDriverBuildBarrier {
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
#[cfg(test)]
type TierDriverTestFactory = Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
pub(crate) type TierDriverTestFactory =
Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
#[cfg(test)]
tokio::task_local! {
static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
pub(crate) static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
}
#[cfg(test)]
@@ -1377,27 +1378,42 @@ async fn ensure_no_authoritative_persisted_references<S>(
where
S: TierReferenceProofStore,
{
ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| {
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
Ok((
journal.tier_name.clone(),
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
))
})
ensure_no_authoritative_persisted_references_with(
api.clone(),
TIER_DELETE_JOURNAL_PREFIX,
"tier-delete journal",
|_object, data| {
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
Ok((
journal.tier_name.clone(),
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
))
},
)
.await?;
ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| {
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
Ok((
transaction.tier_name.clone(),
tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets),
))
})
ensure_no_authoritative_persisted_references_with(
api,
TRANSITION_TRANSACTION_RECORD_PREFIX,
"transition transaction",
|object, data| {
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
Ok((
transaction.tier_name.clone(),
tier_persisted_reference_blocks_any_target(
&transaction.tier_name,
Some(transaction.backend_fingerprint),
targets,
),
))
},
)
.await
}
async fn ensure_no_authoritative_persisted_references_with<S, F>(
api: Arc<S>,
prefix: &str,
reference_kind: &str,
blocks_target: F,
) -> std::result::Result<(), AdminError>
where
@@ -1426,7 +1442,7 @@ where
.map_err(tier_reference_proof_admin_error)?;
let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?;
if blocks {
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name));
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, reference_kind, &object.name));
}
}
if !page.is_truncated {
@@ -1488,16 +1504,21 @@ fn tier_persisted_reference_blocks_target(
fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
let reference_kind = if object.transitioned_object.free_version {
"free-version ownership"
} else {
"transitioned-object"
};
err.message = format!(
"Remote tier {tier_name} still has object references, for example {}/{}",
"Remote tier {tier_name} still has a {reference_kind} reference, for example {}/{}",
object.bucket, object.name
);
err
}
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError {
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, reference_kind: &str, object: &str) -> AdminError {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}");
err.message = format!("Remote tier {tier_name} still has a {reference_kind} reference, for example {object}");
err
}
@@ -3685,13 +3706,17 @@ impl TierConfigMgr {
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let runtime = lock_unpoisoned(&runtime);
if !runtime
let prepared = runtime
.prepared_mutation_blocks
.values()
.any(|blocked_mutation_id| *blocked_mutation_id == mutation_id);
let committed = runtime
.committed_mutation_blocks
.values()
.any(|mutation_ids| mutation_ids.contains(&mutation_id))
{
.any(|mutation_ids| mutation_ids.contains(&mutation_id));
if !prepared && !committed {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier committed mutation fence was not installed".to_string();
err.message = "Remote tier mutation fence was not installed".to_string();
return Err(err);
}
Ok(MutationBlockAllowance {
@@ -3898,14 +3923,6 @@ impl TierConfigMgr {
Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, mutation_block_allowance)
}
fn begin_tier_transition(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
changed: HashSet<String>,
) -> std::result::Result<TierPublishTransition, AdminError> {
Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None)
}
fn begin_tier_transition_with_destinations(
handle: &Arc<RwLock<Self>>,
manager: &mut Self,
@@ -4176,7 +4193,7 @@ impl TierConfigMgr {
let mut config_lock = config_lock;
let coordinated_config_update = config_lock.is_some();
let mut update = Some(update);
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) =
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, target_tiers) =
match mutation {
TierCandidateMutation::Prevalidated(prepared) => {
if version != prepared.version {
@@ -4192,9 +4209,8 @@ impl TierConfigMgr {
)));
}
candidate = prepared.candidate;
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
let mut transition = {
let mut manager = handle.write().await;
let target_tiers = {
let manager = handle.read().await;
let mut target_tiers = changed_tier_names(&manager, &candidate);
if let Some(tier_name) = prepared.explicit_tier_name.as_ref()
&& (manager.tiers.contains_key(tier_name)
@@ -4203,8 +4219,7 @@ impl TierConfigMgr {
{
target_tiers.insert(tier_name.clone());
}
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
.map_err(TierConfigUpdateError::Publish)?
target_tiers
};
(
prepared.kind,
@@ -4212,7 +4227,7 @@ impl TierConfigMgr {
prepared.force,
prepared.current,
prepared.driver_tier,
transition,
target_tiers,
)
}
mutation => {
@@ -4237,11 +4252,9 @@ impl TierConfigMgr {
last_refreshed_at: candidate.last_refreshed_at,
};
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
let mut transition = {
let mut manager = handle.write().await;
let target_tiers = mutation.target_tiers(&manager, &candidate);
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
.map_err(TierConfigUpdateError::Publish)?
let target_tiers = {
let manager = handle.read().await;
mutation.target_tiers(&manager, &candidate)
};
let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline)
.await
@@ -4252,7 +4265,7 @@ impl TierConfigMgr {
mutation_force,
current_for_targets,
driver_tier,
transition,
target_tiers,
)
}
};
@@ -4275,28 +4288,83 @@ impl TierConfigMgr {
save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref())
.await
.map_err(TierConfigUpdateError::Save)?;
let mut blocked_target_tiers = target_tiers.clone();
if let Some(intent) = coordinator_intent.as_ref() {
TierConfigMgr::apply_prepared_mutation_intent_block(&handle, intent)
blocked_target_tiers.extend(intent.affected_targets.iter().map(|target| target.tier_name.clone()));
}
if let Some(intent) = coordinator_intent.as_ref() {
// `target_tiers` may include a stale local-only manager
// entry that is absent from the persisted proof
// snapshot. Fence that local transition under the same
// mutation ID as well; recovery may discard this
// process-local superset, which advances the revision
// and makes the deferred transition fail closed.
TierConfigMgr::apply_prepared_mutation_intent_block_for_tiers(&handle, intent, &blocked_target_tiers)
.await
.map_err(TierConfigUpdateError::Publish)?;
}
let prepared_mutation_block_allowance = match coordinator_intent.as_ref() {
Some(intent) => Some(
TierConfigMgr::mutation_block_allowance_for(&handle, intent.mutation_id)
.await
.map_err(TierConfigUpdateError::Publish)?,
),
None => None,
};
// A durable coordinator intent supplies the admission
// fence that lets us defer generation revocation. Keep the
// original early transition for no-intent paths (for
// example, reconciling a stale local manager to an
// idempotently removed persisted tier), where there is no
// Prepared record capable of blocking a new lease.
let (mut transition, deferred_target_tiers) = if coordinator_intent.is_some() {
(None, Some(target_tiers))
} else {
let transition = {
let mut manager = handle.write().await;
Self::begin_tier_transition_with_destinations(
&handle,
&mut manager,
target_tiers,
HashMap::new(),
None,
)
.map_err(TierConfigUpdateError::Publish)?
};
(Some(transition), None)
};
if coordinated_config_update {
drop(update.take());
drop(config_lock.take());
}
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await {
// The durable Prepared block closes admission before the
// zero-reference proof, but deliberately leaves already
// issued generations current. In particular, an exact
// free-version cleanup that has completed remote DELETE
// must still be able to remove its local ownership marker;
// revoking its generation here would strand that marker
// and make this mutation reject its own interrupted work.
if let Some(intent) = coordinator_intent.as_ref()
&& let Err(drain_error) =
TierConfigMgr::wait_for_blocked_tier_operation_leases_for_tiers(&handle, &blocked_target_tiers).await
{
if !abort_prepared_tier_mutation(&handle, api.clone(), Some(intent), Vec::new()).await {
warn!(
event = "tier_mutation_abort",
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_TIER,
result = "prepared_intent_retained",
coordinator_intent = coordinator_intent.is_some(),
mutation_id = %intent.mutation_id,
"tier mutation lease drain failed and abort was incomplete"
);
}
return Err(TierConfigUpdateError::Publish(drain_error));
} else if let Some(transition) = transition.as_ref() {
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
transition
.wait_for_active_leases_until(drain_deadline)
.await
.map_err(TierConfigUpdateError::Publish)?;
}
let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() {
let peers = match remote_tier_mutation_peers().await {
@@ -4363,6 +4431,71 @@ impl TierConfigMgr {
}
return Err(TierConfigUpdateError::Publish(proof_error));
}
// No affected-tier lease can start after Prepared, and the
// existing set was drained above. It is now safe to revoke
// the generation for publication without invalidating a
// cleanup between its remote and local commit boundaries.
if transition.is_none() {
let target_tiers = deferred_target_tiers.ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier mutation lost its deferred transition targets".to_string();
TierConfigUpdateError::Publish(err)
})?;
let mut manager = handle.write().await;
transition = Some(
match Self::begin_tier_transition_with_destinations(
&handle,
&mut manager,
target_tiers,
HashMap::new(),
prepared_mutation_block_allowance.as_ref(),
) {
Ok(transition) => transition,
Err(transition_error) => {
drop(manager);
if !abort_prepared_tier_mutation(
&handle,
api.clone(),
coordinator_intent.as_ref(),
prepared_peers,
)
.await
{
warn!(
event = "tier_mutation_abort",
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_TIER,
result = "prepared_intent_retained",
coordinator_intent = coordinator_intent.is_some(),
"tier mutation publish transition failed and abort was incomplete"
);
}
return Err(TierConfigUpdateError::Publish(transition_error));
}
},
);
}
let mut transition = transition.ok_or_else(|| {
let mut err = ERR_TIER_INVALID_CONFIG.clone();
err.message = "Remote tier mutation lost its publish transition".to_string();
TierConfigUpdateError::Publish(err)
})?;
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
drop(transition);
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await
{
warn!(
event = "tier_mutation_abort",
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_TIER,
result = "prepared_intent_retained",
coordinator_intent = coordinator_intent.is_some(),
"tier mutation publish drain failed and abort was incomplete"
);
}
return Err(TierConfigUpdateError::Publish(drain_error));
}
let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?;
if coordinated_config_update {
config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await {
@@ -5414,11 +5547,40 @@ impl TierConfigMgr {
handle: &Arc<RwLock<Self>>,
intent: &TierMutationIntent,
) -> std::result::Result<(), AdminError> {
let target_tiers = intent
.affected_targets
.iter()
.map(|target| target.tier_name.clone())
.collect();
Self::apply_prepared_mutation_intent_block_for_tiers(handle, intent, &target_tiers).await
}
async fn apply_prepared_mutation_intent_block_for_tiers(
handle: &Arc<RwLock<Self>>,
intent: &TierMutationIntent,
target_tiers: &HashSet<String>,
) -> std::result::Result<(), AdminError> {
if intent.state != TierMutationIntentState::Prepared {
return Ok(());
}
let manager = handle.read().await;
let runtime = tier_driver_runtime(handle, &manager);
let mut runtime = lock_unpoisoned(&runtime);
let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
for tier_name in target_tiers {
match prepared_mutation_blocks.entry(tier_name.clone()) {
Entry::Vacant(entry) => {
entry.insert(intent.mutation_id);
}
Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
Entry::Occupied(_) => {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = format!("Remote tier {tier_name} already has another prepared mutation");
return Err(err);
}
}
}
if prepared_mutation_blocks == runtime.prepared_mutation_blocks {
return Ok(());
}
@@ -5434,6 +5596,18 @@ impl TierConfigMgr {
pub(crate) async fn wait_for_blocked_tier_operation_leases(
handle: &Arc<RwLock<Self>>,
intent: &TierMutationIntent,
) -> std::result::Result<(), AdminError> {
let target_tiers = intent
.affected_targets
.iter()
.map(|target| target.tier_name.clone())
.collect();
Self::wait_for_blocked_tier_operation_leases_for_tiers(handle, &target_tiers).await
}
async fn wait_for_blocked_tier_operation_leases_for_tiers(
handle: &Arc<RwLock<Self>>,
target_tiers: &HashSet<String>,
) -> std::result::Result<(), AdminError> {
let generations = {
let manager = handle.read().await;
@@ -5441,10 +5615,9 @@ impl TierConfigMgr {
return Ok(());
};
let runtime = lock_unpoisoned(&runtime);
intent
.affected_targets
target_tiers
.iter()
.filter_map(|target| runtime.generations.get(&target.tier_name).cloned())
.filter_map(|tier_name| runtime.generations.get(tier_name).cloned())
.collect::<Vec<_>>()
};
let drain = async {
@@ -14414,6 +14587,13 @@ mod tests {
.push(object);
}
fn remove_listed_version(&self, bucket: &str, object: &str) {
self.listed_versions
.lock()
.expect("tier reference fixture should not poison")
.retain(|version| version.bucket != bucket || version.name != object);
}
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
self.lifecycle_configs
.lock()
@@ -15682,6 +15862,152 @@ mod tests {
);
}
#[tokio::test]
async fn tier_remove_prepared_fence_allows_inflight_free_version_cleanup_to_finish() {
let store = Arc::new(CasConfigStore::default());
let tier = build_rustfs_tier("COLD-A");
let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("free-version drain fixture should persist");
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
guard.tiers.insert("COLD-A".to_string(), tier);
guard.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
guard
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("cleanup")))
.expect("cleanup driver generation should install");
guard
.replace_driver("COLD-B", Box::new(LeaseTestBackend::ready("stale-local")))
.expect("stale local driver generation should install");
}
let cleanup_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("in-flight cleanup lease should be available");
let stale_local_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
.await
.expect("stale local tier lease should be available");
let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(identity));
free_version.transitioned_object.status = "pending".to_string();
free_version.transitioned_object.free_version = true;
store.add_listed_version(free_version);
let remove_manager = manager.clone();
let remove_store = store.clone();
let remove = tokio::spawn(async move {
TIER_MUTATION_TEST_PEERS
.scope(
Vec::new(),
TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true),
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
let prepared = {
let runtime = lock_unpoisoned(&runtime);
runtime.prepared_mutation_blocks.contains_key("COLD-A")
&& runtime.prepared_mutation_blocks.contains_key("COLD-B")
};
if prepared {
break;
}
drop(guard);
tokio::task::yield_now().await;
}
})
.await
.expect("tier remove should install its durable prepared fence");
assert!(
cleanup_lease.is_current(&manager).await,
"the prepared fence must let the already leased cleanup finish its exact local marker deletion"
);
assert!(
stale_local_lease.is_current(&manager).await,
"the local superset fence must also let an already leased stale-manager operation finish"
);
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
Ok(_) => panic!("the prepared fence must reject new tier operations"),
Err(err) => err,
};
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
let stale_blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-B").await {
Ok(_) => panic!("the local superset fence must reject new stale-manager operations"),
Err(err) => err,
};
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&stale_blocked));
store.remove_listed_version("photos", "2026/free-version.jpg");
drop(cleanup_lease);
drop(stale_local_lease);
tokio::time::timeout(Duration::from_secs(5), remove)
.await
.expect("tier remove should finish after both in-flight operations release their leases")
.expect("tier remove task should join")
.expect("tier remove should pass once the in-flight cleanup removes its marker");
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
assert!(!manager.read().await.tiers.contains_key("COLD-B"));
assert!(
!load_tier_config_for_update(store)
.await
.expect("removed tier config should reload")
.0
.tiers
.contains_key("COLD-A")
);
}
#[tokio::test]
async fn no_intent_stale_manager_removal_keeps_early_generation_drain() {
let store = Arc::new(CasConfigStore::default());
empty_mgr()
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("empty persisted tier config should exist");
let manager = TierConfigMgr::new();
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("stale"));
}
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
.await
.expect("stale manager lease should be available");
let remove_manager = manager.clone();
let remove_store = store.clone();
let remove =
tokio::spawn(async move { TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true).await });
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
tokio::task::yield_now().await;
}
})
.await
.expect("no-intent stale-manager reconciliation should revoke before its proof");
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
Ok(_) => panic!("stale-manager reconciliation must not admit a new operation"),
Err(err) => err,
};
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
drop(old);
remove
.await
.expect("stale-manager removal task should join")
.expect("stale-manager removal should converge to the persisted empty config");
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
}
async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
let store = Arc::new(CasConfigStore::default());
let tier = build_rustfs_tier("COLD-A");
@@ -16585,12 +16911,23 @@ mod tests {
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
loop {
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
if prepared {
break;
}
drop(guard);
tokio::task::yield_now().await;
}
})
.await
.expect("owned update should revoke before caller cancellation");
.expect("owned update should install its prepared fence before caller cancellation");
assert!(
old.is_current(&manager).await,
"an already leased operation must remain current until it can finish"
);
caller.abort();
drop(old);
@@ -16635,12 +16972,23 @@ mod tests {
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while old.is_current(&manager).await {
loop {
let guard = manager.read().await;
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
if prepared {
break;
}
drop(guard);
tokio::task::yield_now().await;
}
})
.await
.expect("owned update should revoke before caller cancellation");
.expect("owned update should install its prepared fence before caller cancellation");
assert!(
old.is_current(&manager).await,
"the prepared fence must not invalidate an already leased operation"
);
caller.abort();
let config_file = tier_config_lock_path();
@@ -17026,6 +17374,71 @@ mod tests {
assert!(current.tiers.contains_key("COLD-B"));
}
#[tokio::test]
#[serial_test::serial]
async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
let manager = TierConfigMgr::new();
let store = Arc::new(CasConfigStore::default());
let mut persisted = empty_mgr();
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
persisted
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("prepared-fence revision fixture should persist");
{
let mut guard = manager.write().await;
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
}
let barrier = tier_reference_proof_test_barrier();
let scoped_barrier = barrier.clone();
let update_manager = manager.clone();
let update_store = store.clone();
let update = tokio::spawn(async move {
TIER_REFERENCE_PROOF_TEST_BARRIER
.scope(
scoped_barrier,
TIER_MUTATION_TEST_PEERS.scope(
Vec::new(),
TierConfigMgr::update_candidate_with_config_lock(
&update_manager,
update_store,
TierCandidateMutation::Remove("COLD-A".to_string(), true),
),
),
)
.await
});
barrier.arrived.notified().await;
let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
.await
.expect("an unrelated prepared fence should advance the runtime revision");
barrier.release.add_permits(1);
let err = update
.await
.expect("tier update task should join")
.expect_err("a reference proof cannot authorize publication across a fence revision change");
let TierConfigUpdateError::Publish(err) = err else {
panic!("the stale prepared-fence allowance should fail publication: {err:?}");
};
assert!(err.message.contains("changed before replacement"), "{err}");
assert!(manager.read().await.tiers.contains_key("COLD-A"));
assert!(
load_tier_config_for_update(store)
.await
.expect("rejected tier config should remain readable")
.0
.tiers
.contains_key("COLD-A")
);
TierConfigMgr::clear_prepared_mutation_intent_block(&manager, unrelated.mutation_id)
.await
.expect("unrelated test fence should clear");
}
#[tokio::test]
#[serial_test::serial]
async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() {
+376 -10
View File
@@ -111,10 +111,11 @@ use crate::disk::{
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
use crate::io_support::bitrot::{
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
object_mmap_read_max_length,
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
create_deferred_bitrot_reader_with_stripe_handle,
};
#[cfg(unix)]
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
use crate::set_disk::runtime_sources;
use crate::set_disk::shard_source::ShardReadCost;
use crate::storage_api_contracts::object::ObjectOperations;
@@ -3662,22 +3663,22 @@ async fn rollback_failed_rename(
let object = object.to_string();
let disk_namespace_commit_guard = namespace_commit_guard.clone();
let task = tokio::spawn(async move {
let _namespace_commit_guard = disk_namespace_commit_guard;
let _namespace_commit_guard = disk_namespace_commit_guard.clone();
#[allow(clippy::let_unit_value)]
let _task_guard = SetDisks::rename_fanout_task_guard(&object);
SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
#[cfg(test)]
rollback_fault_injection::before_undo(&object, disk_index)?;
disk.delete_version(
disk.undo_write_with_namespace_owner(
&bucket,
&object,
fi,
false,
DeleteOptions {
undo_write: true,
old_data_dir: rollback_dir,
..Default::default()
},
disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
)
.await
});
@@ -4237,7 +4238,7 @@ impl SetDisks {
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard;
let _namespace_commit_guard = namespace_commit_guard.clone();
let mut dispatch_state = RenameDispatchState::NotDispatched;
let result = std::panic::AssertUnwindSafe(async {
#[allow(clippy::let_unit_value)]
@@ -4272,7 +4273,13 @@ impl SetDisks {
&file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
crate::disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
..Default::default()
},
)
.await;
let rejected_before_publication = observed.rejected_before_publication();
@@ -4601,7 +4608,7 @@ impl SetDisks {
// Keep the storage-owned movement permit attached to the actual
// fan-out owner, even if the caller future is cancelled.
let _fanout_publication_scope = fanout_publication_scope;
let _namespace_commit_guard = fanout_namespace_commit_guard;
let _namespace_commit_guard = fanout_namespace_commit_guard.clone();
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let futures = fanout_disks
@@ -4616,6 +4623,7 @@ impl SetDisks {
let dst_bucket = fanout_dst_bucket.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let publication_scope = scanner_publication_commit_scope.clone();
let namespace_commit_guard = fanout_namespace_commit_guard.clone();
async move {
let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4668,7 +4676,13 @@ impl SetDisks {
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
crate::disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
..Default::default()
},
)
.await;
let rejected_before_publication = observed.rejected_before_publication();
@@ -10859,6 +10873,358 @@ mod tests {
.await;
}
#[cfg(not(windows))]
async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) {
use crate::disk::os;
use futures::FutureExt;
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
],
async {
const DISKS: usize = 4;
let bucket = "namespace-physical-tail";
let object = "inline-overwrite";
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut old = metadata_test_fileinfo(object);
old.mod_time = Some(OffsetDateTime::now_utc());
old.size = 15;
old.parts.clear();
old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None);
old.data = Some(Bytes::from_static(b"old-inline-body"));
old.set_inline_data();
old.metadata.insert("etag".to_string(), "old-etag".to_string());
let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag");
let mut hooks = Vec::new();
let mut entered = Vec::new();
let mut releases = Vec::new();
let mut publication_paths = Vec::new();
for (disk, info) in disks.iter().flatten().zip(&mut infos) {
disk.write_metadata(bucket, bucket, object, old.clone())
.await
.expect("the old inline version must be readable before overwrite");
info.size = 11;
info.parts.clear();
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
let crate::disk::Disk::Local(local) = disk.as_ref() else {
panic!("physical publication fixture requires local disks");
};
// Linux IO paths use a mount FD, which is also the namespace lock key.
let destination = local
.get_disk()
.get_object_path_for_io(bucket, object)
.expect("the publication path must resolve through the disk's mount lease");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
hooks.push(os::prepared_publication_test_hooks::install(
&destination.join(STORAGE_FORMAT_FILE),
move || {
let _ = entered_tx.send(());
// Sender drop also releases the syscall when an earlier assertion fails.
let _ = release_rx.recv();
},
));
entered.push(entered_rx);
releases.push(release_tx);
publication_paths.push(destination);
}
let namespace_owner = ctx.begin_namespace_commit();
let namespace_probe = Arc::downgrade(&namespace_owner);
let receipt = RenameRollbackReceipt::default();
let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
&disks,
(RUSTFS_META_TMP_BUCKET, "source"),
infos,
(bucket, object),
allow_early_ack,
RenameDataFenceOptions::new(3, None)
.with_rollback_receipt(receipt.clone())
.with_namespace_commit_guard(Some(namespace_owner)),
));
tokio::time::timeout(Duration::from_secs(10), async {
tokio::select! {
signals = join_all(entered) => {
assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter");
}
_ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"),
}
})
.await
.expect("all four prepared metadata renames must reach their blocking syscall");
assert!(ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), 1);
// Every wrapper timer exists before advancing; the physical closures stay blocked.
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
let result = tokio::time::timeout(Duration::from_secs(5), rename)
.await
.expect("ordinary disk timeout must not wait for the physical rename");
assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum");
let report = receipt.0.get().expect("failed fanout must finish rollback accounting");
assert_eq!(report.disks.len(), DISKS);
assert!(
report
.disks
.iter()
.all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout)))
);
let pending_before_release = ctx.namespace_commits_pending();
let owner_alive_before_release = namespace_probe.upgrade().is_some();
let old_snapshot_generation = ctx.namespace_commit_generation();
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
let root = disk.path();
assert!(
os::acquire_rename_data_mutation_lease(&root, bucket, destination)
.now_or_never()
.is_none(),
"the physical publication must still own object serialization after the async timeout"
);
assert!(
root.join(RUSTFS_META_TMP_BUCKET)
.join("source")
.join(STORAGE_FORMAT_FILE)
.exists()
);
let stored = disk
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("a scanner can still read the complete old metadata while publication is paused");
assert_eq!(stored.size, 15);
assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice()));
}
assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation);
// Drain real syscalls before checking the regression, including on the RED run.
drop(releases);
for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
let root = disk.path();
let lease = tokio::time::timeout(
Duration::from_secs(5),
os::acquire_rename_data_mutation_lease(&root, bucket, destination),
)
.await
.expect("released physical publishers must drain");
drop(lease);
}
for dir in &dirs {
let reopened = reopen_local_disk(dir).await;
let stored = reopened
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("the detached prepared rename must actually publish after timeout");
assert_eq!(stored.size, 11);
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
// The lease releases its locks before dropping the namespace owner, and the
// owner's `Drop` runs after its `Weak` probe stops upgrading, so wait for the
// pending counter itself instead of asserting it right after the drain.
tokio::time::timeout(Duration::from_secs(5), async {
while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("released physical publishers must release namespace ownership");
let generation_after_publication = ctx.namespace_commit_generation();
assert!(!ctx.namespace_commits_pending());
assert!(namespace_probe.upgrade().is_none());
assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence");
assert!(
pending_before_release && owner_alive_before_release,
"physical publication outlived namespace accounting: early_ack={allow_early_ack}, \
pending={pending_before_release}, owner_alive={owner_alive_before_release}, \
old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}"
);
assert!(
generation_after_publication > old_snapshot_generation,
"physical completion must invalidate the scanner's old metadata snapshot"
);
},
)
.await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() {
assert_namespace_owner_survives_physical_publication_timeout(false).await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() {
assert_namespace_owner_survives_physical_publication_timeout(true).await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() {
use crate::disk::os;
temp_env::async_with_vars(
[
(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
],
async {
let bucket = "physical-ack-tail";
let object = "ack-object";
let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
prepare_rename_source_dirs(&dirs, &disks, "source").await;
let mut infos = rename_commit_fileinfos(object, 4, "new-etag");
for info in &mut infos {
info.size = 11;
info.parts.clear();
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
}
let disk = disks[3].as_ref().expect("tail disk");
let crate::disk::Disk::Local(local) = disk.as_ref() else {
panic!("local fixture");
};
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let owner = ctx.begin_namespace_commit();
let owner_probe = Arc::downgrade(&owner);
let receipt = RenameRollbackReceipt::default();
let caller_receipt = receipt.clone();
let caller_disks = disks.clone();
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let commit = SetDisks::rename_data_owned_with_fence(
&caller_disks,
(RUSTFS_META_TMP_BUCKET, "source"),
infos,
(bucket, object),
true,
RenameDataFenceOptions::new(3, None)
.with_namespace_commit_guard(Some(owner))
.with_rollback_receipt(caller_receipt),
)
.await
.expect("three real disk publications must produce a successful ACK");
assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK");
std::future::pending::<()>().await;
});
let mut commit = tokio::time::timeout(Duration::from_secs(10), async {
entered_rx.await.expect("physical tail entry");
ack_rx
.await
.expect("ACK must arrive while the fourth disk is physically paused")
})
.await
.expect("successful quorum ACK must not wait for its physical tail");
assert_eq!(commit.online_disks.iter().flatten().count(), 3);
assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK");
let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle");
drop(commit);
caller.abort();
assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled());
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain)
.await
.expect("ordinary tail timeout stays bounded after ACK")
.expect("tail owner must not panic")
.expect("successful ACK keeps its convergence result");
assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback");
let pending = ctx.namespace_commits_pending();
let alive = owner_probe.upgrade().is_some();
let generation = ctx.namespace_commit_generation();
for disk in disks.iter().flatten().take(3) {
let stored = disk
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("all ACK voters keep the new object after caller cancellation");
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
drop(release_tx);
let lease = tokio::time::timeout(
Duration::from_secs(5),
os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination),
)
.await
.expect("late physical tail drains");
drop(lease);
tokio::time::timeout(Duration::from_secs(5), async {
while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("late physical tail must release namespace ownership");
for dir in &dirs {
let stored = reopen_local_disk(dir)
.await
.read_version(
"",
bucket,
object,
"",
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("successful ACK remains committed on every disk after late publication");
assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
}
assert!(
pending && alive,
"physical ACK tail must retain namespace ownership after the coordinator exits"
);
assert!(!ctx.namespace_commits_pending());
assert!(owner_probe.upgrade().is_none());
assert!(ctx.namespace_commit_generation() > generation);
assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback");
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {
+7 -1
View File
@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
@@ -4498,6 +4498,12 @@ impl SetDisks {
&self.ctx
}
/// Read the persisted bucket identity through this set's metadata owner.
/// Missing or non-authoritative legacy identities remain errors.
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
}
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
+73 -14
View File
@@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
}
async fn transition_cleanup_store(ctx: &Arc<crate::runtime::instance::InstanceContext>) -> Option<Arc<ECStore>> {
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pause_transition_cleanup_store().await;
transition_object_store(ctx).await
@@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup(
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
#[derive(Default)]
struct TransitionCleanupStoreBarrierState {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
/// One-shot test barrier placed before transition cleanup resolves its ECStore.
pub(crate) struct TransitionCleanupStoreBarrier {
state: Arc<TransitionCleanupStoreBarrierState>,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCleanupStoreBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
impl TransitionCleanupStoreBarrier {
/// Install the process-local barrier for the next cleanup-store resolution.
pub(crate) fn install() -> Self {
@@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier {
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
impl Drop for TransitionCleanupStoreBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier {
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
async fn pause_transition_cleanup_store() {
let barrier = TRANSITION_CLEANUP_STORE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
struct TransitionUploadedCommitBarrierState {
bucket: String,
object: String,
@@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionUploadedCommitBarrier {
state: Arc<TransitionUploadedCommitBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionUploadedCommitBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl TransitionUploadedCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(TransitionUploadedCommitBarrierState {
@@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionUploadedCommitBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) {
let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
upload_cleanup.update_cleanup_transaction(&transaction);
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_after_transition_uploaded_persisted(bucket, object).await;
let commit_opts = opts.as_commit_opts();
@@ -12672,6 +12672,65 @@ mod metadata_mutation_generation_tests {
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
}
#[tokio::test]
#[serial_test::serial(metadata_cache_invalidation_probe)]
async fn segment_observation_equal_size_mutations_retire_metadata_generation() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "segment-observation-bucket";
let object = "hot/object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("create segment fixture bucket");
}
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await;
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
let mut replacement = PutObjReader::from_vec(b"after!".to_vec());
set_disks
.put_object(bucket, object, &mut replacement, &ObjectOptions::default())
.await
.expect("commit same-length replacement with normal owner locking");
assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation");
assert_retired(&set_disks, &old_key).await;
drop(probe);
let after = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("read replacement metadata");
assert_eq!(before.size, after.size);
assert_ne!(before.etag, after.etag, "equal size is not equal content");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read replacement body through the owner");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("drain replacement body");
assert_eq!(body, b"after!");
drop(reader);
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await;
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])),
..Default::default()
},
)
.await
.expect("commit metadata-only mutation with normal owner locking");
assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences");
assert_retired(&set_disks, &old_key).await;
let after = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("read committed metadata-only mutation");
assert_eq!(before.size, after.size);
assert_eq!(before.etag, after.etag);
assert!(!before.user_defined.contains_key("x-amz-meta-segment"));
assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed"));
}
#[tokio::test]
#[serial_test::serial(metadata_cache_invalidation_probe)]
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
+295 -26
View File
@@ -825,6 +825,11 @@ mod tests {
manual_transition_scope_record_object_name, manual_transition_task_object_name,
manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
},
recovery_control::{
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, load_recovery_control, observe_recovery_source,
save_recovery_control_if_absent,
},
tier_delete_journal::{
DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
@@ -844,12 +849,13 @@ mod tests {
},
transition_transaction::{
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity,
TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator, load_transition_transaction_record,
recover_transition_transaction_records, recover_transition_transaction_records_at,
save_transition_transaction_record, save_transition_transaction_record_if_current,
TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
load_transition_transaction_record, recover_transition_transaction_records,
recover_transition_transaction_records_at, save_transition_transaction_record,
save_transition_transaction_record_if_current, transition_recovery_control_id,
transition_transaction_record_object_name,
},
validate_durable_ilm_record,
@@ -19239,6 +19245,105 @@ mod tests {
assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
temp_dir.path(),
"transition-transaction-expired-attempt-budget",
&[4],
))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Versioned,
},
tier_name: "UNUSEDABANDONEDTIER".to_string(),
backend_fingerprint: [7; 32],
not_after_unix_nanos: 1,
})
.expect("transaction should build");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let record_name =
transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
let source = observe_recovery_source(
store.clone(),
&record_name,
crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
)
.await
.expect("transaction source generation should be observable");
let mut control = IlmRecoveryControl::new(
IlmRecoveryControlIdentity {
protocol: IlmRecoveryProtocol::TransitionTransaction,
canonical_source_path: record_name,
stable_operation_identity: transaction.transaction_id.to_string(),
record_class: "transition_transaction_v1".to_string(),
},
source.generation,
IlmRecoveryClassification::Retrying,
2_000_000_000,
IlmRecoveryErrorCode::None,
)
.expect("recovery control should build");
let mut now = 3_000_000_000;
for _ in 1..MAX_RECOVERY_ATTEMPTS {
now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
control
.claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
.expect("abandoned attempt should claim");
control
.record_expired_attempt(now + 1)
.expect("expired attempt should consume retry budget");
now += 2;
}
now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
control
.claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
.expect("final abandoned attempt should claim");
save_recovery_control_if_absent(store.clone(), &control)
.await
.expect("claimed recovery control should persist");
let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
.await
.expect("recovery should account for the expired attempt");
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
.await
.expect("expired recovery control should remain inspectable");
assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
assert!(persisted.control.owner.is_none());
assert_eq!(
transition_transaction_record_count(store).await,
1,
"budget exhaustion must retain the source record"
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -19351,6 +19456,7 @@ mod tests {
),
];
let mut expected_removes = Vec::new();
let mut recovery_control_ids = Vec::new();
for (case, put_version, remote_version, source_mode) in cases {
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
@@ -19388,6 +19494,8 @@ mod tests {
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
recovery_control_ids
.push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
expected_removes.push((transaction.remote_object, put_version));
}
@@ -19403,6 +19511,14 @@ mod tests {
assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
assert_eq!(backend.exact_remove_count(), 2);
assert_eq!(backend.object_count().await, 0);
for control_id in recovery_control_ids {
let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
.await
.expect("completed recovery control should remain inspectable");
assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
assert_eq!(control.control.attempt_count, 1);
assert!(control.control.owner.is_none());
}
let replay = recover_transition_transaction_records(store, 100, None)
.await
@@ -19415,6 +19531,94 @@ mod tests {
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXTERMINALCRASH";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let remote_version = uuid::Uuid::new_v4().to_string();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Versioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(
transaction.fence(),
TransitionTransactionState::Uploaded,
Some(TransitionRemoteVersion::versioned(remote_version.clone())),
)
.expect("transaction should enter uploaded state");
backend.set_put_remote_version(Some(remote_version)).await;
let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
backend
.put(
&transaction.remote_object,
ReaderImpl::Body(candidate.clone()),
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
)
.await
.expect("mock backend should accept candidate");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
let recovery_store = store.clone();
let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
barrier.wait_until_paused().await;
let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
.await
.expect("terminal control should persist before source cleanup");
assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
assert_eq!(backend.object_count().await, 0);
assert_eq!(backend.exact_remove_count(), 1);
recovery.abort();
assert!(
recovery
.await
.expect_err("recovery should be cancelled at the crash boundary")
.is_cancelled()
);
drop(barrier);
let replay = recover_transition_transaction_records(store.clone(), 100, None)
.await
.expect("terminal control should resume source cleanup without another remote delete");
assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
assert_eq!(transition_transaction_record_count(store).await, 0);
assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -19472,6 +19676,8 @@ mod tests {
save_transition_transaction_record(store.clone(), &uploaded)
.await
.expect("transaction record should persist");
let recovery_control_id =
transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
let recovery_store = store.clone();
@@ -19493,11 +19699,17 @@ mod tests {
.expect("recovery should treat the lost CAS as a retained transaction");
assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
assert_eq!(
load_transition_transaction_record(store, uploaded.transaction_id)
load_transition_transaction_record(store.clone(), uploaded.transaction_id)
.await
.expect("newer transaction revision must remain"),
active
);
let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
.await
.expect("lost source CAS should retain a retryable recovery control");
assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
assert_eq!(control.control.consecutive_failure_count, 1);
assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
assert_eq!(backend.remove_count().await, 0);
}
@@ -19799,27 +20011,15 @@ mod tests {
not_after_unix_nanos: 1_780_000_000_000_000_000,
})
.expect("transaction should build");
let uploaded_fence = transaction
transaction
.advance(
transaction.fence(),
TransitionTransactionState::Uploaded,
Some(TransitionRemoteVersion::versioned(remote_version)),
Some(TransitionRemoteVersion::versioned(remote_version.clone())),
)
.expect("transaction should enter uploaded state");
transaction
.mark_cleanup_pending(
uploaded_fence,
TransitionCleanupProof {
transaction_id: transaction.transaction_id,
write_id: transaction.write_id,
remote_object: transaction.remote_object.clone(),
remote_version: transaction.remote_version.clone(),
backend_fingerprint: transaction.backend_fingerprint,
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
},
)
.expect("transaction should enter cleanup pending state");
let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
backend.set_put_remote_version(Some(remote_version)).await;
backend
.put(
&transaction.remote_object,
@@ -19831,6 +20031,8 @@ mod tests {
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let recovery_control_id =
transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
backend.set_remove_failure(true);
let stats = recover_transition_transaction_records(store.clone(), 100, None)
@@ -19846,6 +20048,42 @@ mod tests {
assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.object_count().await, 1);
let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
.await
.expect("failed recovery control should persist");
assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
assert_eq!(control.control.attempt_count, 1);
assert_eq!(control.control.consecutive_failure_count, 1);
assert!(
control
.control
.next_attempt_at_unix_nanos
.is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
);
backend.set_remove_failure(false);
let replay = recover_transition_transaction_records(store.clone(), 100, None)
.await
.expect("recovery before the persisted deadline should be skipped");
assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
let retry_at = control
.control
.next_attempt_at_unix_nanos
.expect("retry deadline should persist");
let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
.await
.expect("recovery at the persisted deadline should retry the advanced source generation");
assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
assert_eq!(backend.exact_remove_count(), 2);
assert_eq!(backend.object_count().await, 0);
assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
.await
.expect("completed retry control should remain inspectable");
assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
assert_eq!(terminal.control.attempt_count, 2);
}
#[cfg(feature = "test-util")]
@@ -20146,6 +20384,10 @@ mod tests {
local_commit_started
.advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
.expect("transaction should enter local commit state");
let upload_started_control_id =
transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
let local_commit_control_id =
transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
backend.set_put_remote_version(Some(remote_version)).await;
for transaction in [&upload_started, &local_commit_started] {
@@ -20176,6 +20418,19 @@ mod tests {
assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
assert_eq!(backend.remove_count().await, 0);
assert_eq!(backend.exact_remove_count(), 0);
let upload_started_control =
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
.await
.expect("upload-started control should persist");
assert_eq!(
upload_started_control.control.classification,
IlmRecoveryClassification::RetainedAmbiguous
);
let local_commit_control =
load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
.await
.expect("local-commit control should persist");
assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
}
#[cfg(feature = "test-util")]
@@ -20514,7 +20769,7 @@ mod tests {
.await;
let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
.await
.expect("unsupported provider recovery should fail closed");
.expect("active unknown ownership should remain fenced before provider recovery");
assert_eq!(
(
unsupported_stats.scanned,
@@ -20523,14 +20778,28 @@ mod tests {
unsupported_stats.failed
),
(1, 0, 1, 0),
"an unsupported provider probe must retain the unknown upload"
"active unknown ownership must retain the upload before the recovery deadline"
);
assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
let recovery_control_id =
transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
assert!(matches!(
load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
Err(Error::ConfigNotFound)
));
assert!(
backend.contains(&transaction.remote_object).await,
"unsupported recovery must not delete the candidate"
"active ownership must not delete the candidate"
);
assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
assert!(
!backend
.op_log()
.await
.iter()
.any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
"active ownership must not probe the provider"
);
assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup");
backend.set_transition_candidate_probe_override(None).await;
let stats =
+2 -2
View File
@@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation(
}
impl ECStore {
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
data_movement_multipart_discovery_counts()
.lock()
@@ -246,7 +246,7 @@ impl ECStore {
.insert(self.id, 0);
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
data_movement_multipart_discovery_counts()
.lock()
@@ -33,6 +33,9 @@ use std::collections::HashMap;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
/// Explicit pending migration; never activates the production writer or GC.
pub mod migration;
// Root-level control files avoid requiring a new directory before the first
// atomic commit. They remain inside the storage owner's metadata volume.
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
@@ -66,6 +69,23 @@ struct Manifest {
}
impl Manifest {
fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
bytes.extend_from_slice(MAGIC);
bytes.push(VERSION);
bytes.extend_from_slice(owner.as_bytes());
bytes.extend_from_slice(&sequence.to_le_bytes());
bytes.extend_from_slice(
&u64::try_from(payload.len())
.map_err(|_| SnapshotError::TooLarge)?
.to_le_bytes(),
);
bytes.extend_from_slice(&Sha256::digest(payload));
bytes.extend_from_slice(&Sha256::digest(&bytes));
Self::decode(&bytes, payload.len())?;
Ok(bytes)
}
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
return Err(SnapshotError::Corrupt);
File diff suppressed because it is too large Load Diff
+772 -11
View File
@@ -18,7 +18,7 @@ use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use time::macros::offset;
use time::{self, Duration, OffsetDateTime};
@@ -65,6 +65,66 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
"Rule with ExpiredObjectDeleteMarker cannot have tags based filtering";
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer";
const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str =
"Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And";
const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates";
const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key";
const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters";
const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative";
const ERR_LIFECYCLE_FILTER_SIZE_RANGE: &str = "ObjectSizeGreaterThan must be smaller than ObjectSizeLessThan";
/// Longest tag key S3 accepts.
const MAX_TAG_KEY_LEN: usize = 128;
/// Longest tag value S3 accepts.
const MAX_TAG_VALUE_LEN: usize = 256;
/// A validation failure that the S3 boundary must answer with `MalformedXML`
/// rather than `InvalidArgument`: the document does not match the published
/// schema shape (wrong number of `Filter` predicates, a one-member `And`).
///
/// Everything else stays [`std::io::ErrorKind::Other`], which the boundary
/// already maps to `InvalidArgument`.
pub const LIFECYCLE_MALFORMED_XML_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
/// A persisted rule that could never have passed validation. Callers that can
/// report an error surface it; evaluation itself stays fail-closed and takes
/// no action for the rule.
pub const LIFECYCLE_CORRUPT_RULE_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
fn malformed_xml_error(message: &'static str) -> std::io::Error {
std::io::Error::new(LIFECYCLE_MALFORMED_XML_ERROR_KIND, message)
}
/// The retention count a rule keeps, or `None` when the persisted value is
/// negative — a shape PUT validation rejects, so reaching it means the rule
/// came from older persistence or an import.
///
/// A negative count must never be read as "retain everything": that is how an
/// invalid configuration silently stopped deleting versions (backlog#2201).
pub fn retained_noncurrent_versions(count: i32) -> Option<usize> {
usize::try_from(count).ok()
}
/// Does any rule carry a retention count that validation would have rejected?
pub fn lifecycle_has_corrupt_retention_count(lc: &BucketLifecycleConfiguration) -> bool {
lc.rules.iter().any(rule_has_corrupt_retention_count)
}
fn rule_has_corrupt_retention_count(rule: &LifecycleRule) -> bool {
let expiration_count = rule
.noncurrent_version_expiration
.as_ref()
.and_then(|expiration| expiration.newer_noncurrent_versions);
let transition_counts = rule
.noncurrent_version_transitions
.iter()
.flatten()
.filter_map(|transition| transition.newer_noncurrent_versions);
expiration_count
.into_iter()
.chain(transition_counts)
.any(|count| retained_noncurrent_versions(count).is_none())
}
pub use rustfs_scanner_metrics::metrics::IlmAction;
@@ -141,6 +201,17 @@ impl RuleValidate for LifecycleRule {
return Err(std::io::Error::other(ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT));
}
if let Some(filter) = self.filter.as_ref() {
validate_lifecycle_filter(filter)?;
}
// A negative retention count was accepted and then read as "retain
// (almost) everything" during evaluation, so an HTTP-accepted rule
// silently stopped deleting versions (backlog#2201).
if rule_has_corrupt_retention_count(self) {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS));
}
// Rule with DelMarkerExpiration cannot have tags based filtering
let has_tag_filter = self
.filter
@@ -173,11 +244,14 @@ impl RuleValidate for LifecycleRule {
// Rule must have at least one action
let has_expiration = self.expiration.is_some();
let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
let has_noncurrent_expiration = self
.noncurrent_version_expiration
.as_ref()
.and_then(|e| e.noncurrent_days)
.is_some();
// `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS
// form: it keeps the newest N noncurrent versions and expires the rest
// with no age condition. RustFS accepts it for MinIO compatibility, so
// it has to count as an action here — otherwise a count-only rule was
// rejected as actionless (backlog#2201).
let has_noncurrent_expiration = self.noncurrent_version_expiration.as_ref().is_some_and(|expiration| {
expiration.noncurrent_days.is_some() || expiration.newer_noncurrent_versions.is_some_and(|count| count > 0)
});
let has_noncurrent_transition = self
.noncurrent_version_transitions
.as_ref()
@@ -203,6 +277,81 @@ impl RuleValidate for LifecycleRule {
}
}
/// Structural validation for `LifecycleRuleFilter`.
///
/// The generated DTO is all-`Option`, so the S3 schema constraints have to be
/// checked here: at most one top-level predicate, an `And` that actually
/// combines at least two, no repeated tag key, tag key/value limits, and a
/// coherent non-negative size range (backlog#2201).
///
/// A filter with no predicate at all stays valid: AWS documents an empty
/// `Filter` as "applies to every object in the bucket", and rejecting it would
/// break the most common way to write an unconditional rule.
fn validate_lifecycle_filter(filter: &LifecycleRuleFilter) -> Result<(), std::io::Error> {
let top_level_predicates = usize::from(filter.prefix.is_some())
+ usize::from(filter.tag.is_some())
+ usize::from(filter.object_size_greater_than.is_some())
+ usize::from(filter.object_size_less_than.is_some())
+ usize::from(filter.and.is_some());
if top_level_predicates > 1 {
return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES));
}
if let Some(tag) = filter.tag.as_ref() {
validate_lifecycle_tag(tag)?;
}
if let Some(and) = filter.and.as_ref() {
let tags = and.tags.as_deref().unwrap_or(&[]);
let and_predicates = usize::from(and.prefix.is_some())
+ tags.len()
+ usize::from(and.object_size_greater_than.is_some())
+ usize::from(and.object_size_less_than.is_some());
if and_predicates < 2 {
return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES));
}
let mut seen_keys = HashSet::with_capacity(tags.len());
for tag in tags {
validate_lifecycle_tag(tag)?;
let key = tag.key.as_deref().unwrap_or_default();
if !seen_keys.insert(key) {
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY));
}
}
validate_lifecycle_size_bounds(and.object_size_greater_than, and.object_size_less_than)?;
}
validate_lifecycle_size_bounds(filter.object_size_greater_than, filter.object_size_less_than)?;
Ok(())
}
/// S3 requires a tag to carry a key and value; both are length-bounded.
/// The DTO makes both optional, so incomplete tags have to be rejected here
/// rather than silently matching nothing.
fn validate_lifecycle_tag(tag: &s3s::dto::Tag) -> Result<(), std::io::Error> {
let key = tag.key.as_deref().unwrap_or_default();
let Some(value) = tag.value.as_deref() else {
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
};
if key.is_empty() || key.chars().count() > MAX_TAG_KEY_LEN || value.chars().count() > MAX_TAG_VALUE_LEN {
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
}
Ok(())
}
fn validate_lifecycle_size_bounds(greater_than: Option<i64>, less_than: Option<i64>) -> Result<(), std::io::Error> {
if greater_than.is_some_and(|size| size < 0) || less_than.is_some_and(|size| size < 0) {
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE));
}
if let (Some(greater_than), Some(less_than)) = (greater_than, less_than)
&& greater_than >= less_than
{
return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_SIZE_RANGE));
}
Ok(())
}
fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> {
// Prefer a non-empty legacy prefix; treat an empty legacy prefix as if it were not set
if let Some(p) = rule.prefix.as_deref()
@@ -293,6 +442,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
{
return true;
}
// A positive count is an action on its own (the MinIO count-only
// form). Zero means "no count constraint" here, exactly as the
// batch limit path reads it, and a negative count is corrupt —
// neither makes the rule active (backlog#2201).
if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions
&& newer_noncurrent_versions > 0
{
@@ -563,6 +716,23 @@ impl Lifecycle for BucketLifecycleConfiguration {
if let Some(ref lc_rules) = self.filter_rules(obj).await {
for rule in lc_rules.iter() {
// A retention count that PUT validation would have rejected can
// only come from older persistence or an import. Take no action
// for the rule instead of allowing another action on the same
// corrupt rule to delete or transition an object (backlog#2201).
if rule_has_corrupt_retention_count(rule) {
debug!(
event = EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object = %obj.name,
rule_id = %rule.id.clone().unwrap_or_default(),
reason = "corrupt_newer_noncurrent_versions",
"Skipped lifecycle evaluation for a rule with an invalid retention count"
);
continue;
}
if obj.is_latest && obj.expired_object_deletemarker() {
if let Some(expiration) = rule.expiration.as_ref()
&& expiration.expired_object_delete_marker.is_some_and(|v| v)
@@ -619,11 +789,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& (noncurrent_version_expiration.noncurrent_days.is_some()
|| noncurrent_version_expiration
.newer_noncurrent_versions
.is_some_and(|count| count > 0))
&& noncurrent_version_expiration
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
{
// A count-only rule (MinIO extension) has no age condition:
// every version past the retained count is due as soon as it
// became noncurrent, i.e. zero days after the successor.
let noncurrent_days = noncurrent_version_expiration.noncurrent_days.unwrap_or(0);
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
@@ -791,15 +968,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
for rule in filter_rules.iter() {
if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
return if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
if newer_noncurrent_versions == 0 {
// Zero means "no count constraint"; a negative count is
// corrupt and must not be read as "retain everything"
// (backlog#2201). Neither yields a limit event.
let Some(retained) = retained_noncurrent_versions(newer_noncurrent_versions).filter(|c| *c > 0) else {
continue;
}
};
Event {
action: IlmAction::DeleteVersionAction,
rule_id: rule.id.clone().unwrap_or_default(),
noncurrent_days: u32::try_from(noncurrent_version_expiration.noncurrent_days.unwrap_or(0))
.unwrap_or(u32::MAX),
newer_noncurrent_versions: usize::try_from(newer_noncurrent_versions).unwrap_or(usize::MAX),
newer_noncurrent_versions: retained,
due: Some(OffsetDateTime::UNIX_EPOCH),
storage_class: "".into(),
}
@@ -1162,7 +1342,11 @@ mod tests {
use super::*;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass};
use s3s::dto::{
LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, NoncurrentVersionTransition,
TransitionStorageClass,
};
use s3s::xml::{Deserialize as XmlDeserialize, SerializeContent as XmlSerializeContent};
use serial_test::serial;
use std::sync::Arc;
use time::macros::datetime;
@@ -4183,6 +4367,583 @@ mod tests {
assert_eq!(event.action, IlmAction::NoneAction);
}
// ---- backlog#2201: retention-count and Filter invariants -----------------
fn rule_with_noncurrent_expiration(expiration: NoncurrentVersionExpiration) -> LifecycleRule {
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent".to_string()),
noncurrent_version_expiration: Some(expiration),
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}
}
fn rule_with_filter(filter: LifecycleRuleFilter) -> LifecycleRule {
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: Some(filter),
id: Some("filtered".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}
}
fn config_with_rules(rules: Vec<LifecycleRule>) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules,
}
}
fn tag(key: &str, value: &str) -> s3s::dto::Tag {
s3s::dto::Tag {
key: Some(key.to_string()),
value: Some(value.to_string()),
}
}
#[tokio::test]
async fn validate_rejects_negative_newer_noncurrent_versions() {
// A negative retention count used to be accepted and then read as
// usize::MAX during evaluation, so the rule silently stopped deleting
// versions (backlog#2201).
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(-1),
})]);
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("a negative retention count must be rejected");
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS);
assert_ne!(err.kind(), LIFECYCLE_MALFORMED_XML_ERROR_KIND, "value errors stay InvalidArgument");
}
#[tokio::test]
async fn validate_rejects_negative_newer_noncurrent_versions_on_transition() {
let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(30),
newer_noncurrent_versions: None,
});
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
newer_noncurrent_versions: Some(-3),
noncurrent_days: Some(1),
storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
}]);
// The transition validator already refuses a negative count, and it runs
// first, so this pins the rejection rather than the message. The gap
// this PR closes is the expiration side, which had no such check.
config_with_rules(vec![rule])
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("a negative retention count on a transition must be rejected");
}
#[tokio::test]
async fn zero_newer_noncurrent_versions_means_no_count_constraint() {
// Zero carries no constraint, matching how the batch limit path has
// always read it. Alongside an age condition the rule is valid; on its
// own it says nothing, so the rule has no action.
config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(0),
})])
.validate(&ObjectLockConfiguration::default())
.await
.expect("zero count alongside NoncurrentDays is valid");
let err = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(0),
})])
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("a zero count on its own is not an action");
assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION);
}
#[tokio::test]
async fn validate_accepts_count_only_noncurrent_expiration() {
// MinIO extension: NewerNoncurrentVersions with no NoncurrentDays. It
// used to be rejected as an actionless rule (backlog#2201).
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(2),
})]);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("a count-only noncurrent expiration rule is accepted");
}
#[tokio::test]
async fn eval_inner_expires_versions_beyond_count_only_retention() {
// Count-only rules have no age condition: everything past the retained
// count is due as soon as it became noncurrent.
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(2),
})]);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
successor_mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
is_latest: false,
num_versions: 5,
..Default::default()
};
// Rank 2 is the third-newest noncurrent version: past a retention of 2.
let expired = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 2).await;
assert_eq!(expired.action, IlmAction::DeleteVersionAction);
assert_eq!(expired.rule_id, "noncurrent");
// Rank 1 is still within the retained count.
let retained = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 1).await;
assert_eq!(retained.action, IlmAction::NoneAction);
}
#[tokio::test]
#[serial]
async fn eval_inner_keeps_age_condition_when_count_and_days_are_set() {
// With both set, the count gates which versions are candidates and the
// age condition still decides when they are due.
with_default_ilm_process_time(|| {});
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(10),
newer_noncurrent_versions: Some(1),
})]);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
is_latest: false,
num_versions: 3,
..Default::default()
};
let too_young = lc.eval_inner(&opts, datetime!(2025-01-05 00:00:00 UTC), 2).await;
assert_eq!(too_young.action, IlmAction::NoneAction, "the age condition still applies");
let due = lc.eval_inner(&opts, datetime!(2025-01-20 00:00:00 UTC), 2).await;
assert_eq!(due.action, IlmAction::DeleteVersionAction);
}
#[tokio::test]
async fn eval_inner_takes_no_action_for_a_corrupt_retention_count() {
// Reachable only from older persistence or an import; it must not be
// read as "retain everything", and it must not delete either.
let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
})]);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
is_latest: false,
num_versions: 3,
..Default::default()
};
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 2).await;
assert_eq!(event.action, IlmAction::NoneAction);
}
#[tokio::test]
async fn eval_inner_does_not_expire_latest_object_for_a_corrupt_retention_rule() {
let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
});
rule.expiration = Some(LifecycleExpiration {
days: Some(1),
..Default::default()
});
let lc = config_with_rules(vec![rule]);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
is_latest: true,
..Default::default()
};
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
assert_eq!(event.action, IlmAction::NoneAction);
}
#[tokio::test]
async fn eval_inner_does_not_delete_latest_marker_for_a_corrupt_retention_rule() {
let mut expired_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
});
expired_marker_rule.expiration = Some(LifecycleExpiration {
expired_object_delete_marker: Some(true),
..Default::default()
});
let mut aged_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
});
aged_marker_rule.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(1) });
for rule in [expired_marker_rule, aged_marker_rule] {
let lc = config_with_rules(vec![rule]);
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
version_id: Some(Uuid::new_v4()),
is_latest: true,
delete_marker: true,
num_versions: 1,
..Default::default()
};
let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
assert_eq!(event.action, IlmAction::NoneAction);
}
}
#[test]
fn corrupt_retention_count_is_detected_on_either_action() {
let mut transition_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(0),
});
transition_rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
newer_noncurrent_versions: Some(-1),
noncurrent_days: Some(1),
storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
}]);
assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
})
])));
assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![transition_rule])));
assert!(!lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(3),
})
])));
}
#[test]
fn count_only_rules_are_active_only_for_a_positive_count() {
let positive = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(2),
})]);
assert!(positive.has_active_rules(""));
let corrupt = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(-1),
})]);
assert!(!corrupt.has_active_rules(""), "a corrupt retention count must not make a rule active");
}
#[tokio::test]
async fn noncurrent_versions_expiration_limit_ignores_a_corrupt_count() {
// The batch path must not read a negative count as "retain everything".
let lc = Arc::new(config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(-1),
})]));
let opts = ObjectOpts {
name: "obj".to_string(),
mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
is_latest: false,
..Default::default()
};
let event = lc.noncurrent_versions_expiration_limit(&opts).await;
assert_eq!(event.action, IlmAction::NoneAction);
assert_eq!(event.newer_noncurrent_versions, 0);
}
#[tokio::test]
async fn validate_covers_filter_invariants() {
struct Case {
name: &'static str,
filter: LifecycleRuleFilter,
expected: Option<(&'static str, std::io::ErrorKind)>,
}
let cases = vec![
Case {
// AWS documents an empty Filter as "every object in the bucket".
name: "empty filter applies to all objects",
filter: LifecycleRuleFilter::default(),
expected: None,
},
Case {
name: "single prefix predicate",
filter: LifecycleRuleFilter {
prefix: Some("logs/".to_string()),
..Default::default()
},
expected: None,
},
Case {
name: "two top-level predicates",
filter: LifecycleRuleFilter {
prefix: Some("logs/".to_string()),
tag: Some(tag("env", "prod")),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
},
Case {
name: "prefix alongside And",
filter: LifecycleRuleFilter {
prefix: Some("logs/".to_string()),
and: Some(LifecycleRuleAndOperator {
prefix: Some("logs/".to_string()),
tags: Some(vec![tag("env", "prod")]),
..Default::default()
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
},
Case {
name: "And with a single member",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
prefix: Some("logs/".to_string()),
..Default::default()
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
},
Case {
name: "And with two members",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
prefix: Some("logs/".to_string()),
tags: Some(vec![tag("env", "prod")]),
..Default::default()
}),
..Default::default()
},
expected: None,
},
Case {
name: "And with two tags",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
tags: Some(vec![tag("env", "prod"), tag("team", "storage")]),
..Default::default()
}),
..Default::default()
},
expected: None,
},
Case {
name: "And repeating a tag key",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
tags: Some(vec![tag("env", "prod"), tag("env", "dev")]),
..Default::default()
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY, std::io::ErrorKind::Other)),
},
Case {
name: "empty tag key",
filter: LifecycleRuleFilter {
tag: Some(tag("", "prod")),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
},
Case {
name: "missing tag key",
filter: LifecycleRuleFilter {
tag: Some(s3s::dto::Tag {
key: None,
value: Some("prod".to_string()),
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
},
Case {
name: "missing tag value",
filter: LifecycleRuleFilter {
tag: Some(s3s::dto::Tag {
key: Some("env".to_string()),
value: None,
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
},
Case {
name: "empty tag value",
filter: LifecycleRuleFilter {
tag: Some(tag("env", "")),
..Default::default()
},
expected: None,
},
Case {
name: "tag key at the limit",
filter: LifecycleRuleFilter {
tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN), "prod")),
..Default::default()
},
expected: None,
},
Case {
name: "tag key past the limit",
filter: LifecycleRuleFilter {
tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN + 1), "prod")),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
},
Case {
name: "tag value past the limit",
filter: LifecycleRuleFilter {
tag: Some(tag("env", &"v".repeat(MAX_TAG_VALUE_LEN + 1))),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
},
Case {
name: "negative ObjectSizeGreaterThan",
filter: LifecycleRuleFilter {
object_size_greater_than: Some(-1),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
},
Case {
name: "negative ObjectSizeLessThan",
filter: LifecycleRuleFilter {
object_size_less_than: Some(-5),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
},
Case {
name: "inverted size range inside And",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
object_size_greater_than: Some(100),
object_size_less_than: Some(100),
..Default::default()
}),
..Default::default()
},
expected: Some((ERR_LIFECYCLE_FILTER_SIZE_RANGE, std::io::ErrorKind::Other)),
},
Case {
name: "valid size range inside And",
filter: LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
object_size_greater_than: Some(1),
object_size_less_than: Some(2),
..Default::default()
}),
..Default::default()
},
expected: None,
},
];
for case in cases {
let result = config_with_rules(vec![rule_with_filter(case.filter)])
.validate(&ObjectLockConfiguration::default())
.await;
match (case.expected, result) {
(None, Ok(())) => {}
(None, Err(err)) => panic!("{}: expected acceptance, got {err}", case.name),
(Some((message, _)), Ok(())) => panic!("{}: expected rejection with {message}", case.name),
(Some((message, kind)), Err(err)) => {
assert_eq!(err.to_string(), message, "{}", case.name);
assert_eq!(err.kind(), kind, "{}: wrong S3 error category", case.name);
}
}
}
}
#[tokio::test]
async fn validate_keeps_legacy_prefix_and_filter_mutually_exclusive() {
let mut rule = rule_with_filter(LifecycleRuleFilter {
prefix: Some("logs/".to_string()),
..Default::default()
});
rule.prefix = Some("legacy/".to_string());
let err = config_with_rules(vec![rule])
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("legacy Prefix and Filter cannot both be present");
assert_eq!(err.to_string(), ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT);
}
#[test]
fn count_only_rule_round_trips_through_xml() {
// The MinIO count-only form has to survive the wire codec, or the rule
// this PR now accepts could not be persisted and read back.
let xml = br#"<LifecycleConfiguration><Rule><ID>count-only</ID><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>2</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
let mut deserializer = s3s::xml::Deserializer::new(xml);
let parsed =
<BucketLifecycleConfiguration as XmlDeserialize>::deserialize(&mut deserializer).expect("count-only XML parses");
let expiration = parsed.rules[0]
.noncurrent_version_expiration
.as_ref()
.expect("noncurrent expiration is present");
assert_eq!(expiration.newer_noncurrent_versions, Some(2));
assert_eq!(expiration.noncurrent_days, None);
let mut buf = Vec::new();
let mut serializer = s3s::xml::Serializer::new(&mut buf);
XmlSerializeContent::serialize_content(&parsed, &mut serializer).expect("count-only config serializes");
let serialized = String::from_utf8(buf).expect("serialized XML is UTF-8");
assert!(
serialized.contains("<NewerNoncurrentVersions>2</NewerNoncurrentVersions>"),
"retention count survives the round trip: {serialized}"
);
assert!(
!serialized.contains("<NoncurrentDays>"),
"a count-only rule must not gain an age condition: {serialized}"
);
}
mod adversarial_regressions {
use super::*;
use s3s::dto::NoncurrentVersionExpiration;
+15 -1
View File
@@ -22,7 +22,10 @@ use rustfs_replication::ReplicationStatusType;
use rustfs_scanner_metrics::metrics::IlmAction;
use crate::object_lock;
use crate::{Event, Lifecycle, ObjectOpts, expiration_action_has_valid_target};
use crate::{
Event, LIFECYCLE_CORRUPT_RULE_ERROR_KIND, Lifecycle, ObjectOpts, expiration_action_has_valid_target,
lifecycle_has_corrupt_retention_count,
};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
@@ -152,6 +155,17 @@ impl Evaluator {
format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()),
));
}
// PUT validation rejects a negative retention count, so a rule that
// carries one came from older persistence or an import. Report it
// instead of evaluating a configuration that cannot be honoured;
// `eval_inner` independently takes no action for such a rule
// (backlog#2201).
if lifecycle_has_corrupt_retention_count(&self.policy) {
return Err(std::io::Error::new(
LIFECYCLE_CORRUPT_RULE_ERROR_KIND,
"lifecycle configuration carries a negative 'NewerNoncurrentVersions'",
));
}
Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await)
}
}
+298
View File
@@ -14,6 +14,7 @@
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
use serde::{Deserialize, Serialize, ser::SerializeMap};
use sha2::{Digest, Sha256};
use std::{
collections::{HashMap, HashSet},
future::Future,
@@ -400,6 +401,85 @@ impl DataUsageScanCheckpoint {
}
}
/// Durable scope of a bucket checkpoint, independent of namespace mutation counters.
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct DataUsageScanIdentity {
pub version: u16,
pub bucket_incarnation: uuid::Uuid,
pub set_layout: DataUsageScanPlanDigest,
pub publication_epoch: u64,
pub tier_registry_generation: u64,
pub scan_mode: HealScanMode,
}
impl Serialize for DataUsageScanIdentity {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(6))?;
map.serialize_entry("version", &self.version)?;
map.serialize_entry("bucket_incarnation", &self.bucket_incarnation)?;
map.serialize_entry("set_layout", &self.set_layout)?;
map.serialize_entry("publication_epoch", &self.publication_epoch)?;
map.serialize_entry("tier_registry_generation", &self.tier_registry_generation)?;
map.serialize_entry("scan_mode", &self.scan_mode)?;
map.end()
}
}
impl DataUsageScanIdentity {
pub(crate) fn is_valid(&self) -> bool {
self.version == 1
&& !self.bucket_incarnation.is_nil()
&& matches!(self.scan_mode, HealScanMode::Normal | HealScanMode::Deep)
}
}
/// A forward coverage sweep may span budgets, but not authorize mixed mutation generations.
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct DataUsageScanProgress {
pub started_plan: DataUsageScanPlanDigest,
pub requested_plan: DataUsageScanPlanDigest,
}
impl Serialize for DataUsageScanProgress {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(2))?;
map.serialize_entry("started_plan", &self.started_plan)?;
map.serialize_entry("requested_plan", &self.requested_plan)?;
map.end()
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct DataUsageScanCoverageReceipt {
pub through: String,
pub digest: [u8; 32],
}
impl Serialize for DataUsageScanCoverageReceipt {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(2))?;
map.serialize_entry("through", &self.through)?;
map.serialize_entry("digest", &self.digest)?;
map.end()
}
}
struct CheckpointDigestWriter(Sha256);
impl std::io::Write for CheckpointDigestWriter {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.update(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct DataUsageEntryInfo {
pub name: String,
@@ -470,6 +550,12 @@ pub struct DataUsageCacheInfo {
#[serde(default)]
pub scan_checkpoint: Option<DataUsageScanCheckpoint>,
#[serde(default)]
pub scan_identity: Option<DataUsageScanIdentity>,
#[serde(default)]
pub scan_progress: Option<DataUsageScanProgress>,
#[serde(default)]
pub scan_coverage_receipt: Option<DataUsageScanCoverageReceipt>,
#[serde(default)]
pub pending_heals: Vec<PendingScannerHeal>,
#[serde(default)]
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
@@ -481,6 +567,11 @@ pub struct DataUsageCacheInfo {
pub snapshot_complete: bool,
#[serde(default)]
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
/// Full activity and inventory scope of a set scan; only a complete
/// snapshot proves coverage. Bucket caches bind this scope into their
/// opaque scan plan digest instead.
#[serde(default)]
pub scan_coverage_digest: Option<DataUsageScanPlanDigest>,
#[serde(default)]
pub cache_key_format: u16,
/// Registry generation used for the completed/partial scan. This is
@@ -517,6 +608,10 @@ impl Serialize for DataUsageCacheInfo {
// Keep this metadata map-encoded so older readers can ignore fields
// appended by newer scanner versions during rolling upgrades.
let field_count = 16
+ usize::from(self.scan_identity.is_some())
+ usize::from(self.scan_progress.is_some())
+ usize::from(self.scan_coverage_receipt.is_some())
+ usize::from(self.scan_coverage_digest.is_some())
+ usize::from(self.tier_registry_generation.is_some())
+ usize::from(!self.size_reconciliation.is_empty())
+ usize::from(self.lkg_snapshot_complete)
@@ -536,11 +631,23 @@ impl Serialize for DataUsageCacheInfo {
state.serialize_entry("failed_objects", &self.failed_objects)?;
state.serialize_entry("scan_resume_after", &self.scan_resume_after)?;
state.serialize_entry("scan_checkpoint", &self.scan_checkpoint)?;
if let Some(identity) = self.scan_identity {
state.serialize_entry("scan_identity", &identity)?;
}
if let Some(progress) = self.scan_progress {
state.serialize_entry("scan_progress", &progress)?;
}
if let Some(receipt) = &self.scan_coverage_receipt {
state.serialize_entry("scan_coverage_receipt", receipt)?;
}
state.serialize_entry("pending_heals", &self.pending_heals)?;
state.serialize_entry("object_lock", &self.object_lock)?;
state.serialize_entry("source", &self.source)?;
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
if let Some(coverage) = self.scan_coverage_digest {
state.serialize_entry("scan_coverage_digest", &coverage)?;
}
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
if let Some(generation) = self.tier_registry_generation {
state.serialize_entry("tier_registry_generation", &generation)?;
@@ -703,6 +810,174 @@ impl DataUsageCache {
}
}
pub(crate) fn prepare_bucket_checkpoint(
&mut self,
name: &str,
next_cycle: u64,
leader_epoch: u64,
source: DataUsageCacheSource,
scan_plan_digest: DataUsageScanPlanDigest,
identity: DataUsageScanIdentity,
) -> DataUsageCachePrepareOutcome {
if self.info.next_cycle > next_cycle {
return DataUsageCachePrepareOutcome::RejectedNewerCycle;
}
if self.info.leader_epoch > leader_epoch {
return DataUsageCachePrepareOutcome::RejectedNewerLeader;
}
let reusable = identity.is_valid()
&& name != DATA_USAGE_ROOT
&& self.info.name == name
&& self.info.source == Some(source)
&& self.info.leader_epoch == leader_epoch
&& self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT
&& self.info.scan_identity == Some(identity)
&& self.info.tier_registry_generation == Some(identity.tier_registry_generation)
&& (self.cache.is_empty() || self.checked_flatten_complete_scope(name).is_some());
if reusable
&& self.info.snapshot_complete
&& self.info.scan_progress.is_none()
&& self.info.scan_checkpoint.is_none()
&& self.info.scan_resume_after.is_none()
&& self.info.scan_coverage_receipt.is_none()
&& self.info.scan_plan_digest == Some(scan_plan_digest)
{
return self.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, true);
}
if !reusable {
let keep_debts = self.info.name == name
&& self
.info
.scan_identity
.is_none_or(|previous| previous.bucket_incarnation == identity.bucket_incarnation);
let (pending_heals, size_reconciliation) = if keep_debts {
(
std::mem::take(&mut self.info.pending_heals),
std::mem::take(&mut self.info.size_reconciliation),
)
} else {
(Vec::new(), HashMap::new())
};
*self = Self::default();
self.info.pending_heals = pending_heals;
self.info.size_reconciliation = size_reconciliation;
}
let cursor_is_valid = (self.info.scan_checkpoint.is_none()
&& self.info.scan_resume_after.is_none()
&& self.info.scan_coverage_receipt.is_none())
|| self.validated_scan_frontier().is_some();
if !cursor_is_valid {
self.info.scan_progress = None;
}
self.info.name = name.to_owned();
self.info.next_cycle = next_cycle;
self.info.leader_epoch = leader_epoch;
self.info.source = Some(source);
self.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
self.info.tier_registry_generation = Some(identity.tier_registry_generation);
self.info.scan_identity = Some(identity);
self.info.snapshot_complete = false;
if let Some(progress) = &mut self.info.scan_progress {
progress.requested_plan = scan_plan_digest;
} else {
self.info.scan_progress = Some(DataUsageScanProgress {
started_plan: scan_plan_digest,
requested_plan: scan_plan_digest,
});
self.info.scan_resume_after = None;
self.info.scan_checkpoint = None;
self.info.scan_coverage_receipt = None;
}
// Old readers do not understand coverage sweeps. An absent plan makes
// their existing prepare path rebuild instead of promoting mixed data.
self.info.scan_plan_digest = None;
if reusable {
DataUsageCachePrepareOutcome::Reused
} else {
DataUsageCachePrepareOutcome::Reset
}
}
fn coverage_prefix_digest(&self, through: &str) -> Result<[u8; 32], serde_json::Error> {
let mut writer = CheckpointDigestWriter(Sha256::new());
serde_json::to_writer(
&mut writer,
&(
&self.info.name,
self.info.scan_identity,
self.info.source,
self.info.leader_epoch,
self.info.cache_key_format,
self.info.scan_progress.map(|progress| progress.started_plan),
through,
),
)?;
let mut prefix = self
.cache
.iter()
.filter(|(key, _)| {
let ancestor = through
.strip_prefix(key.as_str())
.is_some_and(|suffix| suffix.starts_with('/'));
let descendant = key.strip_prefix(through).is_some_and(|suffix| suffix.starts_with('/'));
(key.as_str() <= through && !ancestor) || descendant
})
.collect::<Vec<_>>();
prefix.sort_unstable_by_key(|(key, _)| *key);
for (key, entry) in prefix {
let mut value = serde_json::to_value(entry)?;
value.sort_all_objects();
if let Some(children) = value.get_mut("children").and_then(serde_json::Value::as_array_mut) {
children.sort_unstable_by(|left, right| left.as_str().cmp(&right.as_str()));
}
serde_json::to_writer(&mut writer, &(key, value))?;
}
Ok(writer.0.finalize().into())
}
pub(crate) fn validated_scan_frontier(&self) -> Option<&str> {
let receipt = self.info.scan_coverage_receipt.as_ref()?;
let checkpoint = self.info.scan_checkpoint.as_ref()?;
(self.info.scan_progress.is_some()
&& self.info.scan_identity.is_some_and(|identity| identity.is_valid())
&& self.info.source.is_some()
&& receipt.through.len() <= 16 * 1024
&& checkpoint.version == DATA_USAGE_SCAN_CHECKPOINT_VERSION
&& checkpoint.resume_after == receipt.through
&& self.info.scan_resume_after.as_deref() == Some(receipt.through.as_str())
&& receipt
.through
.strip_prefix(&self.info.name)
.is_some_and(|suffix| suffix.starts_with('/'))
&& self.find(&receipt.through).is_some()
&& self.coverage_prefix_digest(&receipt.through).ok() == Some(receipt.digest))
.then_some(receipt.through.as_str())
}
/// Seal only the frontier supplied by completed traversal, never a restored cursor.
pub(crate) fn seal_scan_frontier(&mut self, frontier: Option<&str>) -> Result<(), serde_json::Error> {
if self.info.scan_progress.is_none() {
self.info.scan_coverage_receipt = None;
return Ok(());
}
let frontier = frontier.filter(|path| path.len() <= 16 * 1024 && self.find(path).is_some());
self.info.scan_coverage_receipt = match frontier {
Some(through) => Some(DataUsageScanCoverageReceipt {
through: through.to_owned(),
digest: self.coverage_prefix_digest(through)?,
}),
None => None,
};
self.info.scan_resume_after = frontier.map(str::to_owned);
let reason = self
.info
.scan_checkpoint
.as_ref()
.map_or(DataUsageScanCheckpointReason::Unknown, |checkpoint| checkpoint.reason);
self.info.scan_checkpoint = frontier.map(|through| DataUsageScanCheckpoint::new(through.to_owned(), reason));
Ok(())
}
fn ensure_cache_save_metrics_registered() {
CACHE_SAVE_METRICS_ONCE.call_once(|| {
describe_counter!(
@@ -782,6 +1057,29 @@ impl DataUsageCache {
(visited == expected_entries).then_some(entry)
}
pub(crate) fn has_complete_root_inventory(&self, bucket_keys: &HashSet<String>) -> bool {
let Some(root) = self.find(DATA_USAGE_ROOT) else {
return false;
};
// Set roots only connect bucket entries. Scalar data at the root, an
// extra bucket, or an orphan must not disappear during bucket folding.
root.children.len() == bucket_keys.len()
&& bucket_keys.iter().all(|key| root.children.contains(key))
&& root.size == 0
&& root.objects == 0
&& root.versions == 0
&& root.delete_markers == 0
&& root.failed_objects == 0
&& !root.compacted
&& root.obj_sizes.is_empty()
&& root.obj_versions.is_empty()
&& root.replication_stats.is_none()
&& root.all_tier_stats.is_none()
&& root.unknown_tier_stats.is_none()
&& root.tier_accounting_proof.is_none()
&& self.checked_flatten_complete(DATA_USAGE_ROOT).is_some()
}
fn checked_flatten_inner(&self, path: &str) -> Option<(DataUsageEntry, usize)> {
let root_key = hash_path(path).key();
let (root_key, root) = self.cache.get_key_value(&root_key)?;
@@ -29,6 +29,31 @@ use tokio::sync::Mutex;
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([3; 32]);
#[test]
fn scoped_scan_coverage_metadata_preserves_map_compatibility() {
#[derive(serde::Deserialize)]
struct LegacyInfo {
name: String,
next_cycle: u64,
}
let mut info = DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: 7,
..Default::default()
};
let old = serde_json::to_value(&info).expect("legacy metadata should encode");
assert!(old.get("scan_coverage_digest").is_none());
let old: DataUsageCacheInfo = serde_json::from_value(old).expect("missing coverage must remain readable");
assert!(old.scan_coverage_digest.is_none());
info.scan_coverage_digest = Some(TEST_PLAN_DIGEST);
let encoded = rmp_serde::to_vec(&info).expect("coverage metadata should remain map encoded");
let legacy: LegacyInfo = rmp_serde::from_slice(&encoded).expect("old map readers should ignore additive proof fields");
assert_eq!(legacy.name, DATA_USAGE_ROOT);
assert_eq!(legacy.next_cycle, 7);
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&encoded).expect("new reader should restore the coverage proof");
assert_eq!(decoded.scan_coverage_digest, Some(TEST_PLAN_DIGEST));
}
#[derive(Debug, PartialEq, Eq)]
struct CachePutRecord {
object: String,
@@ -728,6 +728,15 @@ async fn scan_and_persist_local_bucket(
DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: Some(tier_registry_generation),
checkpoint_identity: crate::scanner_io::scanner_bucket_checkpoint_identity(
&set,
&bucket,
expected_publication_epoch,
tier_registry_generation,
scan_mode,
)
.await
.ok(),
},
);
match scan_state {
@@ -821,6 +830,22 @@ async fn scan_and_persist_local_bucket(
ScannerDiskScanOutcome::Partial(cache) => (cache, Some(RemoteScannerFrameResult::Partial)),
ScannerDiskScanOutcome::NamespaceNotFound(cache) => (cache, Some(RemoteScannerFrameResult::NamespaceNotFound)),
};
if let Some(expected) = cache.info.scan_identity
&& crate::scanner_io::scanner_bucket_checkpoint_identity(
&set,
&bucket,
expected_publication_epoch,
tier_registry_generation,
scan_mode,
)
.await
.ok()
!= Some(expected)
{
return Err(RemoteScannerServerError::retry_bucket(
"remote scanner checkpoint identity changed during scanning",
));
}
if guard.is_lock_lost() {
return Err(RemoteScannerServerError::worker(
+52 -26
View File
@@ -1086,6 +1086,12 @@ impl ScannerMaintenanceFeatures {
fn needs_regular_cycle(self) -> bool {
self.lifecycle || self.replication || self.inspection_failed
}
fn requires_full_scan(self, observed_generation: Option<u64>, current_generation: u64, wake: ScannerCycleWakeReason) -> bool {
self.needs_regular_cycle()
|| observed_generation != Some(current_generation)
|| !matches!(wake, ScannerCycleWakeReason::DirtyUsage | ScannerCycleWakeReason::ClusterActivity)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -1283,18 +1289,18 @@ async fn configure_scanner_defaults(
ctx: &CancellationToken,
storeapi: &Arc<impl ScannerStorage>,
) -> (ScannerMaintenanceFeatures, Option<u64>) {
let (features, maintenance_generation) = detect_stable_scanner_maintenance_features(ctx, storeapi)
.await
.unwrap_or_else(|| {
(
ScannerMaintenanceFeatures {
inspection_failed: true,
..Default::default()
},
scanner_maintenance_generation(),
)
});
if storeapi.setup_is_erasure_sd().await {
let (features, maintenance_generation) = detect_stable_scanner_maintenance_features(ctx, storeapi)
.await
.unwrap_or_else(|| {
(
ScannerMaintenanceFeatures {
inspection_failed: true,
..Default::default()
},
scanner_maintenance_generation(),
)
});
// Single-disk keeps the speed-preset-derived default cycle (60s at the
// `default` preset) instead of a special shorter cycle: no measured
// cold-start ILM latency basis for an override, and clean-idle backoff
@@ -1319,7 +1325,7 @@ async fn configure_scanner_defaults(
} else {
set_scanner_default_speed(ScannerSpeed::Default);
set_scanner_default_cycle_secs(None);
(ScannerMaintenanceFeatures::default(), None)
(features, Some(maintenance_generation))
}
}
@@ -1564,7 +1570,7 @@ where
S: ScannerStorage,
{
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget, true).await
}
#[instrument(skip_all)]
@@ -1576,6 +1582,7 @@ async fn run_data_scanner_cycle_with_budget<S>(
cycle_revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_budget: Arc<ScannerCycleBudget>,
requires_full_scan: bool,
) -> ScannerCycleOutcome
where
S: ScannerStorage,
@@ -1714,6 +1721,9 @@ where
scan_mode,
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
persisted_usage_baseline: usage_persist_baseline.data.clone(),
requires_full_scan,
#[cfg(test)]
resolved_scope_observer: None,
},
)
.await;
@@ -1878,9 +1888,9 @@ where
// A remote restart or movement flip invalidates
// the token proof; usage_store interprets this
// as a publication barrier and performs no PUT.
return true;
return Some(ScannerCycleDeferReason::DataMovement);
}
storeapi.scanner_data_usage_publication_blocked().await
scanner_local_publication_defer_reason(storeapi.as_ref()).await
}
},
)
@@ -2574,10 +2584,7 @@ where
let mut superseded_backoff = ScannerRetryBackoff::default();
let mut deferred_backoff = ScannerRetryBackoff::default();
let initial_runtime_config = resolve_scanner_runtime_config();
if clean_idle_topology_supported
&& scanner_clean_idle_backoff_configured(&initial_runtime_config)
&& maintenance_generation_seen.is_none()
{
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
global_metrics().set_cycle(None).await;
finish_scanner_leader_iteration(false, "stopped", String::new()).await;
@@ -2785,6 +2792,7 @@ where
&mut cycle_revision,
leader_epoch,
cycle_budget.clone(),
true,
),
guard.lock_lost_notified(),
)
@@ -2879,7 +2887,7 @@ where
#[cfg(test)]
notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
let runtime_config = resolve_scanner_runtime_config();
if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) {
if clean_idle_topology_supported {
let current_generation = scanner_maintenance_generation();
if maintenance_generation_seen != Some(current_generation) {
scanner_activity_seen = None;
@@ -3075,6 +3083,11 @@ where
&mut cycle_revision,
leader_epoch,
cycle_budget.clone(),
maintenance_features.requires_full_scan(
maintenance_generation_seen,
scanner_maintenance_generation(),
wake_reason,
),
),
guard.lock_lost_notified(),
)
@@ -3132,10 +3145,7 @@ where
let maintenance_config_changed =
maintenance_generation_seen.is_some_and(|generation| generation != current_maintenance_generation);
let retry_failed_inspection = maintenance_inspection_retry.retry_due(maintenance_features, wake_reason, Instant::now());
if clean_idle_topology_supported
&& scanner_clean_idle_backoff_configured(&runtime_config)
&& (maintenance_config_changed || retry_failed_inspection)
{
if clean_idle_topology_supported && (maintenance_config_changed || retry_failed_inspection) {
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
break;
};
@@ -3229,8 +3239,8 @@ where
{
match status {
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
if storeapi.scanner_data_usage_publication_blocked().await {
return Some(ScannerCycleDeferReason::DataMovement);
if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await {
return Some(reason);
}
if status == ScannerCycleStatus::Complete {
let distributed = storeapi.setup_is_dist_erasure().await;
@@ -3253,6 +3263,22 @@ where
}
}
async fn scanner_local_publication_defer_reason<S>(storeapi: &S) -> Option<ScannerCycleDeferReason>
where
S: ScannerStorage,
{
if !storeapi.scanner_data_usage_publication_blocked().await {
return None;
}
// Pending namespace commits invalidate this publication attempt, but only
// storage movement creates durable, rate-limited catch-up debt.
if storeapi.scanner_data_movement_pause_status().await.paused {
Some(ScannerCycleDeferReason::DataMovement)
} else {
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
}
}
fn scanner_post_lease_activity_defer_reason(
expected_digest: Option<[u8; 32]>,
activity: Result<ScannerActivitySnapshot, String>,
+197 -14
View File
@@ -266,6 +266,11 @@ async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
}
let pause_status = store.scanner_data_movement_pause_status().await;
assert!(pause_status.paused);
assert_eq!(
scanner_local_publication_defer_reason(store.as_ref()).await,
Some(ScannerCycleDeferReason::DataMovement),
"an actual data-movement pause must retain durable catch-up tracking"
);
paused_probe.wait().await;
drop(paused_probe);
@@ -1171,6 +1176,9 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now())
.await
.expect("scanner pause backlog should be available");
let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default())
@@ -1195,6 +1203,13 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
.await
.expect("fixture usage baseline should be readable");
let pending = ecstore_hold_namespace_commit(store.as_ref());
assert_eq!(
scanner_local_publication_defer_reason(store.as_ref()).await,
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
"an ordinary namespace commit must not be classified as data movement"
);
let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
assert_eq!(pause_backlog_attempt, ScannerPauseBacklogAttemptDecision::Untracked);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
let mut cycle_info = CurrentCycle {
@@ -1204,12 +1219,20 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
let mut revision = DataUsageCacheRevision::Missing;
let outcome = tokio::time::timeout(
Duration::from_secs(30),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget)),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&budget), true),
)
.await
.expect("the coordinator must finish its namespace walk while a PUT is pending");
assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal");
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(
outcome,
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
finish_scanner_pause_backlog_cycle(&mut pause_backlog, &store, pause_backlog_attempt, outcome).await;
let pause_backlog_status = scanner_pause_backlog_status(store.clone()).await;
assert_eq!(pause_backlog_status.phase, ScannerPauseBacklogPhase::Idle);
assert!(!pause_backlog_status.pending_full_scan);
assert_eq!(pause_backlog_status.catch_up_attempts, 0);
assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle");
assert_eq!(revision, DataUsageCacheRevision::Missing);
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
@@ -1240,7 +1263,7 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
let outcome = tokio::time::timeout(
Duration::from_secs(30),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget)),
run_data_scanner_cycle_with_budget(&ctx, &store, &mut cycle_info, &mut revision, 1, Arc::clone(&retry_budget), true),
)
.await
.expect("the same cycle must converge after the pending PUT drains");
@@ -4869,6 +4892,28 @@ async fn usage_bootstrap_does_not_overwrite_concurrent_replacement() {
#[serial]
async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let quota_ledger_path = "config/quota-ledger/reserved-bucket.json";
let quota_ledger = serde_json::to_vec(&serde_json::json!({
"version": 1,
"bucket_incarnation": "00000000-0000-0000-0000-000000000001",
"quota_revision_unix_nanos": 1,
"accounted_usage": 100,
"reservations": {
"00000000-0000-0000-0000-000000000002": {
"object": "pending-object",
"old_size": 0,
"new_size": 64,
"created_at": 1,
"pool_index": 0,
"set_index": 0,
"commit_started": true
}
}
}))
.expect("quota ledger fixture should encode");
save_config(store.clone(), quota_ledger_path, quota_ledger.clone())
.await
.expect("independent quota reservations should persist");
let cycle = CurrentCycle {
current: 41,
next: 42,
@@ -4927,6 +4972,14 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
assert_eq!(usage.scanner_epoch, Some(9));
assert_eq!(
read_config(store.clone(), quota_ledger_path)
.await
.expect("quota ledger must remain readable after scanner reset"),
quota_ledger,
"scanner reset must preserve incarnation and outstanding reserved bytes exactly"
);
for path in [
usage_backup_path.as_str(),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -5796,7 +5849,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier
let probe_calls = route_probe_calls.clone();
async move {
let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
route_blocked && call > 1
(route_blocked && call > 1).then_some(ScannerCycleDeferReason::DataMovement)
}
},
)
@@ -5842,16 +5895,19 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(
outcome,
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert!(!store.objects.lock().await.contains_key(&target_key));
assert_eq!(
store.put_counts.lock().await.get(&target_key),
None,
"the final pool-state fence must run before the first PUT"
"the final publication fence must run before the first PUT"
);
}
}
@@ -5872,7 +5928,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing(
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -5920,7 +5976,7 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -5961,7 +6017,7 @@ async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() {
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -6021,7 +6077,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
data: Some(Bytes::from(snapshot_data)),
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::DataMovement) },
)
.await;
@@ -6061,7 +6117,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() {
// Model the remote lease holder flipping its movement generation
// after the activity probe but before the coordinator's PUT.
route_store.publication_admission_blocked.store(true, Ordering::Release);
false
None
}
},
)
@@ -6099,7 +6155,7 @@ async fn coordinator_classifies_an_expired_publication_lease() {
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(None, Some(expired), None),
|| async { false },
|| async { None },
)
.await;
@@ -6178,7 +6234,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::DataMovement) },
)
.await;
@@ -8208,6 +8264,76 @@ fn clean_idle_backoff_policy_preserves_explicit_and_maintenance_cycles() {
}
}
#[tokio::test]
#[serial]
async fn scoped_scan_explicit_bitrot_keeps_dirty_planning_without_idle_backoff() {
temp_env::async_with_vars([(ENV_SCANNER_CYCLE, None), (ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"))], async {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let (_temp_dir, store) = setup_scanner_cycle_store_with_pool_count(true, 2).await;
let ctx = CancellationToken::new();
let (features, generation) = configure_scanner_defaults(&ctx, &store).await;
let config = resolve_scanner_runtime_config();
assert_eq!(config.cycle_interval_source, ScannerRuntimeConfigSource::Default);
assert_eq!(config.bitrot_cycle_source, ScannerRuntimeConfigSource::Env);
assert!(!scanner_clean_idle_backoff_configured(&config));
assert!(!features.needs_regular_cycle());
assert_eq!(
generation,
Some(scanner_maintenance_generation()),
"multi-disk startup must inspect maintenance independently"
);
let observed = ScannerCycleObservedGenerations::for_wait(&config, None, 7, 0, scanner_maintenance_generation());
assert_eq!(observed.dirty_usage, Some(7), "explicit bitrot still permits ordinary dirty wakeups");
for (wake, full) in [
(ScannerCycleWakeReason::DirtyUsage, false),
(ScannerCycleWakeReason::ClusterActivity, false),
(ScannerCycleWakeReason::Timer, true),
(ScannerCycleWakeReason::ClusterMaintenance, true),
] {
assert_eq!(
features.requires_full_scan(generation, scanner_maintenance_generation(), wake),
full,
"{wake:?}"
);
}
assert!(features.requires_full_scan(None, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage));
for unsafe_features in [
ScannerMaintenanceFeatures {
lifecycle: true,
..Default::default()
},
ScannerMaintenanceFeatures {
replication: true,
..Default::default()
},
ScannerMaintenanceFeatures {
inspection_failed: true,
..Default::default()
},
] {
assert!(unsafe_features.requires_full_scan(
generation,
scanner_maintenance_generation(),
ScannerCycleWakeReason::DirtyUsage
));
}
crate::scanner_io::record_scanner_maintenance_change("maintenance-proof-change");
assert!(features.requires_full_scan(generation, scanner_maintenance_generation(), ScannerCycleWakeReason::DirtyUsage));
let (refreshed, refreshed_generation) = detect_stable_scanner_maintenance_features(&ctx, &store)
.await
.expect("changed maintenance generation should be inspected");
assert!(!refreshed.requires_full_scan(
Some(refreshed_generation),
scanner_maintenance_generation(),
ScannerCycleWakeReason::DirtyUsage
));
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
})
.await;
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
}
#[test]
fn clean_idle_backoff_requires_activity_probes() {
let default_config = ScannerRuntimeConfig::default();
@@ -8596,6 +8722,63 @@ fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_gen
}
}
#[test]
fn scoped_scan_remote_dirty_coverage_invalidates_local_bucket_current() {
let before = BTreeMap::from([("remote".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut after = before.clone();
let remote = after.get_mut("remote").expect("remote activity should exist");
remote.dirty_usage_generation += 1;
remote.dirty_usage_pending = true;
assert_eq!(scanner_activity_structural_digest(&before), scanner_activity_structural_digest(&after));
let old_plan = crate::scanner_io::checkpoint_fixture_bucket_digest(
DataUsageScanPlanDigest(scanner_activity_snapshot_digest(&before)),
None,
);
let new_plan = crate::scanner_io::checkpoint_fixture_bucket_digest(
DataUsageScanPlanDigest(scanner_activity_snapshot_digest(&after)),
None,
);
assert_ne!(
old_plan, new_plan,
"remote dirty changes must fence Current even without a local bucket hint"
);
let source = DataUsageCacheSource::new(0, 0);
let mut cache = DataUsageCache {
info: crate::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
leader_epoch: 11,
source: Some(source),
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
snapshot_complete: true,
scan_plan_digest: Some(old_plan),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.replace("bucket", "", DataUsageEntry::default());
assert!(matches!(
crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut cache,
"bucket",
source,
7,
11,
new_plan,
crate::scanner_io::DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: None,
checkpoint_identity: None,
},
),
crate::scanner_io::DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
..
}
));
}
#[test]
fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acquisition() {
let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
+20 -18
View File
@@ -265,7 +265,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
receiver,
leader_epoch,
initial_baseline,
|| async { false },
|| async { None },
)
.await
}
@@ -280,7 +280,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx,
@@ -308,7 +308,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx,
@@ -336,7 +336,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
let ScannerPublicationFence {
expected_publication_epoch,
@@ -374,18 +374,19 @@ where
} else {
DATA_USAGE_OBJ_NAME_PATH.as_str()
};
if route_probe().await {
if let Some(reason) = route_probe().await {
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_blocked_before_reconcile",
"Scanner data usage publication deferred by the pool-state fence"
reason = reason.as_str(),
path = %target_path,
"Scanner data usage publication deferred by the publication fence"
);
global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str());
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
global_metrics().record_scanner_usage_deferred(reason.as_str());
outcome = DataUsagePersistOutcome::Deferred(reason);
break;
}
@@ -626,17 +627,18 @@ where
if ctx.is_cancelled() {
break 'updates;
}
if route_probe().await {
if let Some(reason) = route_probe().await {
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_blocked_before_save",
"Scanner data usage publication deferred by the final pool-state fence"
reason = reason.as_str(),
path = %target_path,
"Scanner data usage publication deferred by the final publication fence"
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break DataUsagePersistOutcome::Deferred(reason);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
@@ -722,19 +724,19 @@ where
);
}
Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
let route_blocked = route_probe().await;
if route_blocked {
if let Some(reason) = route_probe().await {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_deferred",
reason = reason.as_str(),
path = %target_path,
error = %e,
"Scanner data usage route is blocked by data movement; retrying later"
"Scanner data usage route remains blocked; retrying later"
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break DataUsagePersistOutcome::Deferred(reason);
}
error!(
target: "rustfs::scanner",
+85 -7
View File
@@ -707,6 +707,9 @@ pub struct FolderScanner {
/// next scan and cannot mix generations in one aggregate.
tier_registry: TierRegistrySnapshot,
pending_heals_changed: bool,
coverage_frontier: Option<String>,
resume_frontier: Option<String>,
coverage_gap: bool,
pending_size_reconciliation_keys: HashSet<String>,
pending_size_reconciliation_scopes: HashSet<String>,
pending_size_reconciliation_truncated: bool,
@@ -906,6 +909,16 @@ impl FolderScanner {
self.update_cache.info.scan_checkpoint = Some(checkpoint);
}
fn record_completed_child(&mut self, folder: &str, healthy: bool) {
if self.old_cache.info.scan_progress.is_some() {
self.coverage_gap |= !healthy;
if !self.coverage_gap {
self.coverage_frontier = Some(folder.to_owned());
}
}
self.record_scan_resume_hint(folder);
}
fn record_scan_resume_hint_if_not_ancestor(&mut self, folder: &str) {
let keep_existing = self
.new_cache
@@ -1281,6 +1294,8 @@ impl FolderScanner {
}
Err(e) => return Err(ScannerError::Io(e)),
};
#[cfg(test)]
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
@@ -1471,6 +1486,7 @@ impl FolderScanner {
// (e.g. in the get_size error branch below). This branch only accounts
// for subsequent skips of already-failed paths.
if self.should_skip_failed(&item.path) {
self.coverage_gap |= self.old_cache.info.scan_progress.is_some();
continue;
}
@@ -1484,6 +1500,7 @@ impl FolderScanner {
let failure_action = classify_get_size_failure(&item, &e);
if failure_action != GetSizeFailureAction::Skip {
self.coverage_gap |= self.old_cache.info.scan_progress.is_some();
// Track failed objects to prevent infinite retry loops
into.failed_objects += 1;
self.record_failed(&item.path);
@@ -1576,6 +1593,9 @@ impl FolderScanner {
abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()]));
apply_scanner_size_summary(into, &sz);
if !sz.size_reconciliation.is_empty() {
self.coverage_gap |= self.old_cache.info.scan_progress.is_some();
}
self.apply_size_reconciliation(&sz);
into.objects += 1;
object_count += 1;
@@ -1620,6 +1640,7 @@ impl FolderScanner {
}
if self.is_erasure_mode && found_erasure_data_directory && !found_object_metadata {
self.coverage_gap |= self.old_cache.info.scan_progress.is_some();
found_object_metadata = true;
let metadata_path = path_join_buf(&[&dir_path, STORAGE_FORMAT_FILE]);
@@ -1741,7 +1762,13 @@ impl FolderScanner {
source: FolderScanSource::Existing,
}));
let has_queued_folders = !queued_folders.is_empty();
let resume_order = order_queued_folders_for_resume(&mut queued_folders, scan_resume_after);
let forward_sweep = self.old_cache.info.scan_progress.is_some();
let forward_resume_after = self.resume_frontier.clone();
let resume_order = if forward_sweep {
order_queued_folders_for_resume(&mut queued_folders, None)
} else {
order_queued_folders_for_resume(&mut queued_folders, scan_resume_after)
};
if checkpoint_tracks_child_order && has_queued_folders {
match resume_order {
FolderResumeOrder::Used => global_metrics().record_scanner_checkpoint_used(),
@@ -1758,6 +1785,18 @@ impl FolderScanner {
let mut folder_item = queued_folder.folder;
let h = hash_path(&folder_item.name);
if forward_sweep
&& !into.compacted
&& forward_resume_after.as_deref().is_some_and(|resume| {
folder_item.name.as_str() <= resume
&& !matches!(folder_resume_match(&folder_item.name, resume), Some(FolderResumeMatch::Descendant))
})
&& self.old_cache.find(&folder_item.name).is_some()
{
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
continue;
}
match queued_folder.source {
FolderScanSource::New => {
@@ -1789,7 +1828,7 @@ impl FolderScanner {
}
}
FolderScanSource::Existing => {
if !into.compacted && self.old_cache.is_compacted(&h) {
if !forward_sweep && !into.compacted && self.old_cache.is_compacted(&h) {
let next_cycle = self.old_cache.info.next_cycle as u32;
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
// Transfer and add as child...
@@ -1810,7 +1849,7 @@ impl FolderScanner {
// In compacted mode child totals are accumulated directly into the parent entry.
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), into));
fut.await.map_err(|e| ScannerError::Other(e.to_string()))?;
self.record_scan_resume_hint(&folder_item.name);
self.record_completed_child(&folder_item.name, into.failed_objects == 0);
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
tokio::task::yield_now().await;
} else {
@@ -1835,12 +1874,13 @@ impl FolderScanner {
error = %e,
"Scanner child folder scan failed"
);
self.coverage_gap |= forward_sweep;
continue;
}
tokio::task::yield_now().await;
into.add_child(&h);
self.record_scan_resume_hint(&folder_item.name);
self.record_completed_child(&folder_item.name, dst.failed_objects == 0);
// We scanned a folder, optionally send update.
self.update_cache.delete_recursive(&h);
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
@@ -2250,7 +2290,10 @@ impl FolderScanner {
self.new_cache.replace_hashed(&this_hash, &folder.parent, into);
}
// Keep independently accounted children while the sweep cursor may
// reference them; the hard cardinality compaction below still applies.
if !into.compacted
&& self.old_cache.info.scan_progress.is_none()
&& self.new_cache.info.name != folder.name
&& let Some(mut flat) = self.new_cache.size_recursive(&this_hash.key())
{
@@ -2359,6 +2402,7 @@ pub async fn scan_data_folder(
cache.fold_retired_tiers(&tier_registry.names);
cache.info.tier_registry_generation = Some(tier_registry.generation);
let resume_frontier = cache.validated_scan_frontier().map(str::to_owned);
// Create folder scanner
let mut scanner = FolderScanner {
root: base_path,
@@ -2388,6 +2432,9 @@ pub async fn scan_data_folder(
local_disk,
tier_registry,
pending_heals_changed: false,
coverage_frontier: resume_frontier.clone(),
resume_frontier,
coverage_gap: false,
pending_size_reconciliation_keys: HashSet::new(),
pending_size_reconciliation_scopes: HashSet::new(),
pending_size_reconciliation_truncated: false,
@@ -2418,23 +2465,41 @@ pub async fn scan_data_folder(
match scanner.scan_folder(ctx.clone(), folder, &mut root).await {
Ok(()) => {
// Get the new cache and finalize it
let coverage_gap = scanner.coverage_gap;
let new_cache = scanner.as_mut_new_cache();
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
new_cache.info.last_update = Some(SystemTime::now());
new_cache.info.next_cycle = cache.info.next_cycle;
let unresolved_objects = root.failed_objects > 0
let unresolved_objects = coverage_gap
|| root.failed_objects > 0
|| !new_cache.info.failed_objects.is_empty()
|| !new_cache.info.size_reconciliation.is_empty();
new_cache.info.snapshot_complete = !unresolved_objects;
let mixed_coverage = new_cache
.info
.scan_progress
.is_some_and(|progress| progress.started_plan != progress.requested_plan);
new_cache.info.snapshot_complete = !unresolved_objects && !mixed_coverage;
if let Some(progress) = &mut new_cache.info.scan_progress {
if new_cache.info.snapshot_complete {
new_cache.info.scan_plan_digest = Some(progress.requested_plan);
new_cache.info.scan_progress = None;
} else {
// Retain observations, then verify from the beginning under
// the latest plan. A clean tail cannot certify an old prefix.
progress.started_plan = progress.requested_plan;
new_cache.info.scan_plan_digest = None;
}
}
let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some();
new_cache.info.scan_resume_after = None;
new_cache.info.scan_checkpoint = None;
new_cache.info.scan_coverage_receipt = None;
if had_scan_checkpoint {
global_metrics().record_scanner_checkpoint_cleared();
}
close_disk_guard.close().await;
if unresolved_objects {
if unresolved_objects || mixed_coverage {
Err(ScannerError::PartialCache(Box::new(new_cache.clone())))
} else {
Ok(new_cache.clone())
@@ -2448,6 +2513,7 @@ pub async fn scan_data_folder(
if root_has_progress {
scanner.carry_forward_old_children(&root_hash, &mut root);
}
let coverage_frontier = scanner.coverage_frontier.clone();
let new_cache = scanner.as_mut_new_cache();
if root_has_progress {
new_cache.replace_hashed(&root_hash, &None, &root);
@@ -2462,6 +2528,18 @@ pub async fn scan_data_folder(
if root_has_progress {
set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason()));
}
new_cache.seal_scan_frontier(coverage_frontier.as_deref())?;
if new_cache.info.scan_progress.is_some() {
if let Some(checkpoint) = &new_cache.info.scan_checkpoint {
global_metrics().record_scanner_checkpoint_set(
checkpoint.version,
checkpoint.resume_after.clone(),
checkpoint.reason.as_str(),
);
} else {
global_metrics().record_scanner_checkpoint_cleared();
}
}
close_disk_guard.close().await;
return Err(ScannerError::PartialCache(Box::new(new_cache.clone())));
}
@@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::Mutex;
mod checkpoint_fixture;
pub(super) mod enumeration_restart;
/// Reset the process-global alert cooldown map; test-only.
fn reset_alert_cooldowns() {
@@ -346,6 +347,9 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
refresh_failed: false,
},
pending_heals_changed: false,
coverage_frontier: None,
resume_frontier: None,
coverage_gap: false,
pending_size_reconciliation_keys: HashSet::new(),
pending_size_reconciliation_scopes: HashSet::new(),
pending_size_reconciliation_truncated: false,
@@ -20,6 +20,8 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
mod segment_observation;
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
const STATIC_OBJECTS: u64 = 24;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
@@ -234,6 +236,531 @@ fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
);
}
fn bound_checkpoint() -> (DataUsageCache, crate::DataUsageScanIdentity) {
let identity = crate::DataUsageScanIdentity {
version: 1,
bucket_incarnation: Uuid::from_u128(1),
set_layout: DataUsageScanPlanDigest([41; 32]),
publication_epoch: 0,
tier_registry_generation: 7,
scan_mode: HealScanMode::Normal,
};
let mut cache = DataUsageCache::default();
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
cache.replace("bucket", "", DataUsageEntry::default());
cache.replace(
"bucket/static",
"bucket",
DataUsageEntry {
objects: 3,
..Default::default()
},
);
cache.info.scan_resume_after = Some("bucket/static".into());
cache.info.scan_checkpoint = Some(DataUsageScanCheckpoint::new(
"bucket/static".into(),
DataUsageScanCheckpointReason::Objects,
));
cache
.seal_scan_frontier(Some("bucket/static"))
.expect("completed fixture prefix receipt");
(cache, identity)
}
#[test]
fn checkpoint_fixture_roundtrip_retains_verified_scope_but_old_reader_rebuilds() {
let (cache, identity) = bound_checkpoint();
let mut cache = decode_fixture(&cache.marshal_msg().expect("encode bound progress")).expect("read bound progress");
let next_plan = DataUsageScanPlanDigest([42; 32]);
assert_eq!(
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, next_plan, identity),
crate::DataUsageCachePrepareOutcome::Reused
);
assert_eq!(retained(&cache), 3);
assert_eq!(cache.info.scan_identity, Some(identity));
assert_eq!(
cache.info.scan_progress,
Some(crate::DataUsageScanProgress {
started_plan: PLAN,
requested_plan: next_plan
})
);
assert!(cache.info.scan_plan_digest.is_none());
let mut old_wire = serde_json::to_value(&cache).expect("map-encoded compatibility fixture");
let old_info = old_wire["info"].as_object_mut().expect("cache info is a map");
old_info.remove("scan_identity");
old_info.remove("scan_progress");
old_info.remove("scan_coverage_receipt");
let mut old_view: DataUsageCache = serde_json::from_value(old_wire).expect("old writer drops unknown metadata");
assert_eq!(
old_view.prepare_for_scan("bucket", 11, 7, SOURCE, next_plan, true),
crate::DataUsageCachePrepareOutcome::Reset
);
assert!(old_view.cache.is_empty());
assert!(!old_view.info.snapshot_complete);
}
#[test]
fn checkpoint_fixture_optional_coverage_metadata_roundtrips_together() {
let (mut cache, _) = bound_checkpoint();
cache.info.scan_coverage_digest = Some(DataUsageScanPlanDigest([77; 32]));
let encoded = cache.marshal_msg().expect("encode every optional coverage field");
let decoded = DataUsageCache::unmarshal(&encoded).expect("decode combined W03/W04 metadata map");
assert_eq!(decoded.info.scan_identity, cache.info.scan_identity);
assert_eq!(decoded.info.scan_progress, cache.info.scan_progress);
assert_eq!(decoded.info.scan_coverage_receipt, cache.info.scan_coverage_receipt);
assert_eq!(decoded.info.scan_coverage_digest, cache.info.scan_coverage_digest);
assert_eq!(decoded.validated_scan_frontier(), Some("bucket/static"));
assert!(!decoded.info.snapshot_complete);
}
#[test]
fn checkpoint_fixture_unchanged_complete_plan_keeps_existing_rescan_policy() {
let (mut cache, identity) = bound_checkpoint();
cache.info.scan_progress = None;
cache.info.scan_plan_digest = Some(PLAN);
cache.info.scan_resume_after = None;
cache.info.scan_checkpoint = None;
cache.info.scan_coverage_receipt = None;
cache.info.snapshot_complete = true;
assert_eq!(
cache.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, PLAN, identity),
crate::DataUsageCachePrepareOutcome::Reused
);
assert!(
cache.info.scan_progress.is_none(),
"unchanged complete coverage needs no forced verification sweep"
);
assert_eq!(cache.info.scan_plan_digest, Some(PLAN));
assert_eq!(retained(&cache), 3);
let next = DataUsageScanPlanDigest([44; 32]);
cache.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, next, identity);
assert_eq!(cache.info.scan_progress.expect("changed plan must be verified").started_plan, next);
assert!(cache.info.scan_plan_digest.is_none());
assert!(!cache.info.snapshot_complete);
}
#[test]
fn checkpoint_fixture_identity_changes_and_future_state_fail_closed() {
let (cache, identity) = bound_checkpoint();
for next_identity in [
crate::DataUsageScanIdentity {
bucket_incarnation: Uuid::from_u128(2),
..identity
},
crate::DataUsageScanIdentity {
set_layout: DataUsageScanPlanDigest([9; 32]),
..identity
},
crate::DataUsageScanIdentity {
publication_epoch: 1,
..identity
},
crate::DataUsageScanIdentity {
tier_registry_generation: 8,
..identity
},
crate::DataUsageScanIdentity {
scan_mode: HealScanMode::Deep,
..identity
},
] {
let mut next = cache.clone();
assert_eq!(
next.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, next_identity),
crate::DataUsageCachePrepareOutcome::Reset
);
assert!(next.cache.is_empty());
assert!(next.info.scan_checkpoint.is_none());
assert!(!next.info.snapshot_complete);
}
for (source, epoch) in [(crate::DataUsageCacheSource::new(1, 0), 7), (SOURCE, 8)] {
let mut next = cache.clone();
assert_eq!(
next.prepare_bucket_checkpoint("bucket", 11, epoch, source, PLAN, identity),
crate::DataUsageCachePrepareOutcome::Reset
);
assert!(next.cache.is_empty());
}
for (cycle, epoch, expected) in [
(10, 7, crate::DataUsageCachePrepareOutcome::RejectedNewerCycle),
(11, 6, crate::DataUsageCachePrepareOutcome::RejectedNewerLeader),
] {
let mut next = cache.clone();
assert_eq!(next.prepare_bucket_checkpoint("bucket", cycle, epoch, SOURCE, PLAN, identity), expected);
assert_eq!(
serde_json::to_value(&next).expect("current cache"),
serde_json::to_value(&cache).expect("saved cache")
);
}
for invalid in [
crate::DataUsageScanIdentity { version: 2, ..identity },
crate::DataUsageScanIdentity {
bucket_incarnation: Uuid::nil(),
..identity
},
] {
let mut next = cache.clone();
crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut next,
"bucket",
SOURCE,
11,
7,
PLAN,
crate::scanner_io::DataUsageCacheReuseOptions {
checkpoint_identity: Some(invalid),
..Default::default()
},
);
assert!(next.cache.is_empty(), "unsupported identity must not retain coverage");
}
}
#[test]
fn checkpoint_fixture_corrupt_cursor_restarts_validation_without_claiming_completion() {
let (cache, identity) = bound_checkpoint();
for resume in ["other/static", "bucket/missing"] {
let mut next = cache.clone();
next.info.scan_resume_after = Some(resume.into());
next.info.scan_checkpoint = Some(DataUsageScanCheckpoint::new(resume.into(), DataUsageScanCheckpointReason::Objects));
let plan = DataUsageScanPlanDigest([43; 32]);
next.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, plan, identity);
assert_eq!(retained(&next), 3, "observations may survive an invalid cursor");
assert!(next.info.scan_resume_after.is_none());
assert!(next.info.scan_checkpoint.is_none());
assert_eq!(next.info.scan_progress.expect("new verification sweep").started_plan, plan);
assert!(!next.info.snapshot_complete);
assert!(next.info.scan_plan_digest.is_none());
}
}
#[test]
fn checkpoint_fixture_receipt_binds_covered_prefix_not_unvisited_suffix() {
let (mut cache, _) = bound_checkpoint();
cache.replace(
"bucket/z-unvisited",
"bucket",
DataUsageEntry {
objects: 99,
..Default::default()
},
);
assert_eq!(cache.validated_scan_frontier(), Some("bucket/static"));
let saved = decode_fixture(&cache.marshal_msg().expect("persist receipt")).expect("load receipt");
assert_eq!(saved.validated_scan_frontier(), Some("bucket/static"));
cache.cache.get_mut("bucket/static").expect("covered prefix").objects = 100;
assert!(
cache.validated_scan_frontier().is_none(),
"altered covered content must invalidate the receipt"
);
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete() {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for (object, size) in [
("a-done/object", 1),
("b-pending/first", 7),
("b-pending/second", 1),
("z-stale/object", 2),
] {
write_checkpoint_object(&root, object, &[(None, size)]).await;
}
let identity = crate::DataUsageScanIdentity {
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
..bound_checkpoint().1
};
for tamper_receipt_path in [false, true] {
let store = FixtureStore::new();
let mut cache = DataUsageCache::default();
let revisions = cache
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("empty fixture revisions");
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
cache.replace("bucket", "", DataUsageEntry::default());
for (prefix, objects) in [("a-done", 1), ("b-pending", 99), ("z-stale", 99)] {
cache.replace(
&format!("bucket/{prefix}"),
"bucket",
DataUsageEntry {
objects,
size: objects,
compacted: true,
..Default::default()
},
);
}
cache
.seal_scan_frontier(Some("bucket/a-done"))
.expect("actual completed prefix receipt");
cache.info.scan_resume_after = Some("bucket/z-stale".into());
cache.info.scan_checkpoint = Some(DataUsageScanCheckpoint::new(
"bucket/z-stale".into(),
DataUsageScanCheckpointReason::Objects,
));
if tamper_receipt_path {
cache.info.scan_coverage_receipt.as_mut().expect("receipt").through = "bucket/z-stale".into();
}
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("persist corrupted existing cursor");
let mut loaded = store.strict_load().await;
let revisions = loaded
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("corrupt cursor CAS revision");
assert!(loaded.validated_scan_frontier().is_none());
loaded.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
assert!(loaded.info.scan_resume_after.is_none());
loaded.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
let outcome = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
loaded,
None,
HealScanMode::Normal,
)
.await
.expect("scan must revisit the prefix");
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
panic!("uncovered suffix must not become complete")
};
assert_eq!(budget.progress().0, 2);
assert_eq!(cache.checked_flatten("bucket/b-pending").expect("revisited prefix").size, 7);
assert!(!cache.info.snapshot_complete);
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("persist verified partial");
assert!(!store.strict_load().await.info.snapshot_complete);
}
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_failed_child_prevents_receipt_advancing_past_gap() {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for object in ["a-good", "b-skipped", "c-later"] {
write_checkpoint_object(&root, object, &[(None, 1)]).await;
}
let identity = crate::DataUsageScanIdentity {
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
..bound_checkpoint().1
};
let mut cache = DataUsageCache::default();
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
cache.info.skip_healing = true;
cache.info.failed_objects.insert(
root.join("bucket/b-skipped/xl.meta").to_string_lossy().into_owned(),
FolderScanner::now_secs(),
);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("scan with a known failed child");
let ScannerDiskScanOutcome::Partial(cache) = result else {
panic!("skipped failure is not complete")
};
assert_eq!(cache.validated_scan_frontier(), Some("bucket/a-good"));
assert!(!cache.info.failed_objects.is_empty());
assert!(!cache.info.snapshot_complete);
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_complete_sampling_partial_resumes_with_fixed_budget() {
check_complete_sampling_resumption(HealScanMode::Normal).await;
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_normal_partial_reenters_prefix_for_deep_scan() {
check_complete_sampling_resumption(HealScanMode::Deep).await;
}
async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for index in 0..9 {
write_checkpoint_object(&root, &format!("prefix/{index:04}"), &[(None, 1)]).await;
}
let identity = crate::DataUsageScanIdentity {
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
..bound_checkpoint().1
};
let mut cache = DataUsageCache::default();
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
cache.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
// Seed an existing complete baseline; every recovery attempt below is bounded.
let baseline = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("initial complete baseline");
let ScannerDiskScanOutcome::Complete(mut cache) = baseline else { panic!("baseline is complete") };
let mut deep_current = cache.clone();
let deep_identity = crate::DataUsageScanIdentity {
scan_mode: HealScanMode::Deep,
..identity
};
let state = crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut deep_current,
"bucket",
SOURCE,
11,
7,
PLAN,
crate::scanner_io::DataUsageCacheReuseOptions {
checkpoint_identity: Some(deep_identity),
..Default::default()
},
);
assert!(
matches!(state, crate::scanner_io::DataUsageCacheScanState::Prepared { .. }),
"Normal complete is not Deep Current"
);
cache.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, PLAN, identity);
assert!(cache.info.scan_progress.is_none(), "complete unchanged baseline uses existing sampling");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(3),
..Default::default()
},
);
let outcome = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("sampling interruption");
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
panic!("sampling must exhaust the three-object budget")
};
assert!(cache.info.scan_progress.is_none());
assert_eq!(cache.info.scan_plan_digest, Some(PLAN));
assert!(cache.info.scan_checkpoint.is_some());
let store = FixtureStore::new();
let mut loaded = DataUsageCache::default();
let revisions = loaded
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("fixture revision");
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("persist sampling partial");
if resume_mode == HealScanMode::Deep {
write_checkpoint_object(&root, "prefix/0000", &[(None, 7)]).await;
}
let resumed_identity = crate::DataUsageScanIdentity {
scan_mode: resume_mode,
..identity
};
for round in 0..16 {
let mut loaded = DataUsageCache::default();
let revisions = loaded
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("reload partial each recovery round");
loaded.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, PLAN, resumed_identity);
assert!(loaded.info.scan_progress.is_some(), "sampling partial must enter forward validation");
loaded.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(3),
..Default::default()
},
);
let result = scanner
.local_disk
.clone()
.nsscanner_disk(budget.token(), budget, vec![scanner.local_disk.clone()], loaded, None, resume_mode)
.await
.expect("bounded recovery scan");
let (cache, complete) = match result {
ScannerDiskScanOutcome::Partial(cache) => (cache, false),
ScannerDiskScanOutcome::Complete(cache) => (cache, true),
_ => panic!("fixture namespace remains present"),
};
if round == 0 && resume_mode == HealScanMode::Deep {
assert_eq!(cache.find("bucket/prefix/0000").expect("Deep revisits earlier prefix").size, 7);
}
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("save bounded recovery");
if complete {
let root = store.strict_load().await.checked_flatten("bucket").expect("certified root");
assert_eq!(root.objects, 9);
assert_eq!(root.size, if resume_mode == HealScanMode::Deep { 15 } else { 9 });
return;
}
}
panic!("sampling interruption must recover with the unchanged three-object budget");
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_save_reload_resume() {
@@ -242,23 +769,48 @@ async fn checkpoint_fixture_save_reload_resume() {
#[tokio::test]
#[serial]
async fn checkpoint_fixture_hot_digest_diagnostic() {
async fn checkpoint_fixture_hot_digest_retains_partial_progress() {
run_checkpoint_fixture(true).await;
}
async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
let mut metadata = FileMeta::new();
for (index, (version_id, size)) in versions.iter().enumerate() {
let mut info = FileInfo::new(object, 4, 2);
info.volume = "bucket".into();
info.version_id = *version_id;
info.versioned = version_id.is_some();
info.size = *size;
info.mod_time = Some(
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("fixture index"))
.expect("non-sentinel fixture modification time"),
);
metadata.add_version(info).expect("fixture version");
}
write_test_object_metadata_bytes(root, "bucket", object, &metadata.marshal_msg().expect("fixture metadata")).await;
}
async fn run_checkpoint_fixture(change_digest: bool) {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for index in 0..STATIC_OBJECTS {
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
write_checkpoint_object(&root, &format!("static/{index:04}"), &[(None, 1)]).await;
}
let identity = crate::DataUsageScanIdentity {
version: 1,
bucket_incarnation: Uuid::from_u128(1),
set_layout: DataUsageScanPlanDigest([41; 32]),
publication_epoch: 0,
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
scan_mode: HealScanMode::Normal,
};
let store = FixtureStore::new();
let mut previous = 0;
let mut visited = 0;
for round in 0..3_u8 {
write_test_object_metadata(&root, "bucket", "hot/current").await;
write_checkpoint_object(&root, "hot/current", &[(None, 1)]).await;
let mut cache = DataUsageCache::default();
let revisions = cache
.load_with_revisions(store.clone(), CACHE_NAME)
@@ -278,9 +830,11 @@ async fn run_checkpoint_fixture(change_digest: bool) {
crate::scanner_io::DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: None,
checkpoint_identity: Some(identity),
},
);
let prepared = retained(&cache);
cache.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
@@ -326,13 +880,12 @@ async fn run_checkpoint_fixture(change_digest: bool) {
eprintln!(
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
);
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
assert_eq!(
diagnosis,
CoverageDiagnosis::Progress,
"visited growth must produce durable static coverage"
);
}
assert_eq!(
diagnosis,
CoverageDiagnosis::Progress,
"visited growth must produce durable static coverage"
);
assert!(loaded.info.scan_plan_digest.is_none(), "old readers must rebuild an uncertified sweep");
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
previous = reloaded;
}
@@ -383,28 +936,85 @@ async fn run_checkpoint_fixture(change_digest: bool) {
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
loaded,
None,
HealScanMode::Normal,
)
store.reject_save.store(false, Ordering::SeqCst);
write_checkpoint_object(&root, "static/0000", &[(Some(Uuid::from_u128(2)), 7), (Some(Uuid::from_u128(3)), 3)]).await;
tokio::fs::remove_dir_all(root.join("bucket/static/0001"))
.await
.expect("unbounded scan must complete after durable partial progress");
let ScannerDiskScanOutcome::Complete(cache) = result else {
panic!("unbounded fixture must produce a complete disk cache");
};
assert!(cache.info.snapshot_complete);
assert!(cache.info.scan_checkpoint.is_none());
assert_eq!(
cache.checked_flatten("bucket").expect("complete bucket root").objects,
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
);
.expect("remove previously scanned fixture object");
write_checkpoint_object(&root, "hot/later", &[(None, 1)]).await;
let final_plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, Some(3));
let mut saw_mixed_sweep_end = false;
for _ in 0..32 {
let mut cache = DataUsageCache::default();
let revisions = cache
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("reload every bounded round");
crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut cache,
"bucket",
SOURCE,
11,
7,
final_plan,
crate::scanner_io::DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: Some(identity.tier_registry_generation),
checkpoint_identity: Some(identity),
},
);
cache.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(4),
..Default::default()
},
);
let outcome = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("bounded sweep outcome");
let (cache, complete) = match outcome {
ScannerDiskScanOutcome::Complete(cache) => (cache, true),
ScannerDiskScanOutcome::Partial(cache) => (cache, false),
ScannerDiskScanOutcome::NamespaceNotFound(_) => panic!("fixture namespace exists"),
};
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("save each bounded sweep");
let saved = store.strict_load().await;
if complete {
assert!(
saw_mixed_sweep_end,
"a clean tail must first finish as partial before a new validation sweep"
);
assert!(saved.info.snapshot_complete);
assert!(saved.info.scan_progress.is_none());
assert!(saved.info.scan_checkpoint.is_none());
assert_eq!(saved.info.scan_plan_digest, Some(final_plan));
let total = saved.checked_flatten("bucket").expect("complete bucket root");
assert_eq!((total.objects, total.versions, total.size), (25, 2, 34));
assert_eq!(saved.checked_flatten("bucket/static").expect("static subtree").objects, 23);
assert_eq!(saved.checked_flatten("bucket/hot").expect("hot subtree").objects, 2);
return;
}
assert!(!saved.info.snapshot_complete);
assert!(saved.info.scan_plan_digest.is_none());
if !budget.budget_elapsed() {
saw_mixed_sweep_end = true;
}
}
panic!("finite stable fixture must converge using the same four-object budget without an unbounded final sweep");
}
@@ -0,0 +1,202 @@
//! Fixture-only range diagnostics. No result is supplied to a scan selector.
use super::*;
use std::collections::BTreeSet;
const MAX_SEGMENTS: usize = 4;
const MAX_SEGMENT_BYTES: usize = 128;
const MAX_WALK_SAMPLES: usize = 32;
const MAX_WALK_BYTES: usize = 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ProposalError {
EntryLimit,
ByteLimit,
InvalidKey,
}
// Keys come from successful fixture writes, not a production mutation stream.
fn fixture_proposal(keys: &[&str]) -> Result<BTreeSet<String>, ProposalError> {
let mut segments = BTreeSet::new();
let mut bytes = 0;
for key in keys {
if key.is_empty() || key.contains(['\\', '\0']) || key.split('/').any(|part| matches!(part, "" | "." | "..")) {
return Err(ProposalError::InvalidKey);
}
let segment = key.split('/').next().expect("validated nonempty key");
if segments.contains(segment) {
continue;
}
if segments.len() == MAX_SEGMENTS {
return Err(ProposalError::EntryLimit);
}
if segment.len() > MAX_SEGMENT_BYTES - bytes {
return Err(ProposalError::ByteLimit);
}
bytes += segment.len();
segments.insert(segment.to_string());
}
Ok(segments)
}
#[test]
fn segment_observation_fixture_proposal_bounds() {
assert_eq!(fixture_proposal(&["hot/one", "hot/two"]), Ok(BTreeSet::from(["hot".to_string()])));
assert_eq!(fixture_proposal(&["a", "b", "c", "d"]).expect("entry boundary").len(), MAX_SEGMENTS);
assert_eq!(fixture_proposal(&["a", "b", "c", "d", "e"]), Err(ProposalError::EntryLimit));
let exact = "x".repeat(MAX_SEGMENT_BYTES);
assert!(fixture_proposal(&[&exact]).is_ok());
assert_eq!(fixture_proposal(&[&exact, "y"]), Err(ProposalError::ByteLimit));
let oversized = "x".repeat(MAX_SEGMENT_BYTES + 1);
assert_eq!(fixture_proposal(&[&oversized]), Err(ProposalError::ByteLimit));
for key in ["", "/hot", "hot/../cold", "hot//one", "hot\\one", "hot/\0"] {
assert_eq!(fixture_proposal(&[key]), Err(ProposalError::InvalidKey));
}
}
fn cache_value(cache: &DataUsageCache) -> serde_json::Value {
let mut value = serde_json::to_value(cache).expect("serialize the entire cache");
// Children are a HashSet: canonicalize only that unordered field, without
// discarding any cache fields or changing ordered histogram arrays.
for (path, entry) in &cache.cache {
value["cache"][path]["children"] =
serde_json::to_value(entry.children.iter().collect::<BTreeSet<_>>()).expect("canonical child set");
}
value
}
async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
let (mut scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for prefix in ["hot", "cold", "other"] {
for leaf in ["one", "two"] {
let object = format!("{prefix}/{leaf}");
let mut metadata = FileMeta::new();
let mut info = FileInfo::new(&object, 4, 2);
info.volume = "bucket".to_string();
info.name = object.clone();
info.size = 1;
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
info.metadata.insert("etag".to_string(), "before".to_string());
metadata.add_version(info).expect("construct segment fixture metadata");
write_test_object_metadata_bytes(&root, "bucket", &object, &metadata.marshal_msg().expect("encode metadata")).await;
}
}
let changed_key = "hot/one";
let changed_path = root.join("bucket").join(changed_key).join("xl.meta");
let before = tokio::fs::read(&changed_path).await.expect("read initial hot metadata");
let mut metadata = FileMeta::new();
let mut info = FileInfo::new(changed_key, 4, 2);
info.volume = "bucket".to_string();
info.name = changed_key.to_string();
info.size = 1;
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
info.metadata.insert("etag".to_string(), "after!".to_string());
metadata.add_version(info).expect("construct same-size hot mutation");
write_test_object_metadata_bytes(&root, "bucket", changed_key, &metadata.marshal_msg().expect("encode hot mutation")).await;
let after = tokio::fs::read(&changed_path)
.await
.expect("read back committed fixture mutation");
assert_eq!(before.len(), after.len(), "fixture rewrite must keep metadata byte length unchanged");
assert_ne!(before, after, "a changed key requires an observable successful fixture write");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
let paths = Arc::new(Mutex::new(Vec::<String>::new()));
let proposed_walked = Arc::new(Mutex::new(BTreeSet::<String>::new()));
scanner.update_current_path = Arc::new({
let paths = paths.clone();
let proposed_walked = proposed_walked.clone();
move |path: &str| {
let mut paths = paths.lock().expect("lock bounded actual-walk samples");
assert!(paths.len() < MAX_WALK_SAMPLES, "fixture walk exceeded its entry budget");
let bytes: usize = paths.iter().map(String::len).sum();
assert!(path.len() <= MAX_WALK_BYTES - bytes, "fixture walk exceeded its byte budget");
paths.push(path.to_string());
if observe {
let proposed = fixture_proposal(&[changed_key]).expect("bounded successful fixture mutation");
if let Some(segment) = path.strip_prefix("bucket/").and_then(|path| path.split('/').next())
&& proposed.contains(segment)
{
proposed_walked
.lock()
.expect("lock bounded observed segments")
.insert(segment.to_string());
}
}
Box::pin(async {})
}
});
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
},
&mut DataUsageEntry::default(),
)
.await
.expect("actual folder walker must finish independently of diagnostics");
let paths = paths.lock().expect("read walk samples").clone();
assert!(!paths.is_empty());
for prefix in ["hot", "cold", "other"] {
assert!(
paths.iter().any(|path| path == &format!("bucket/{prefix}")),
"all fixture segments must actually be walked"
);
}
let store = FixtureStore::new();
let revisions = DataUsageCache::default()
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("read empty fixture revisions");
scanner
.new_cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("save actual walker output through the cache codec and revision gate");
let loaded = store.strict_load().await;
assert_eq!(loaded.checked_flatten("bucket").expect("complete fixture tree").objects, 6);
assert_eq!(
cache_value(&loaded),
cache_value(&scanner.new_cache),
"codec round-trip must retain the entire cache, not just aggregate size"
);
if observe {
let proposed = proposed_walked.lock().expect("read callback observations").clone();
assert_eq!(proposed, BTreeSet::from(["hot".to_string()]));
let walked_segments: BTreeSet<_> = paths
.iter()
.filter_map(|path| path.strip_prefix("bucket/"))
.filter_map(|path| path.split('/').next())
.collect();
assert_eq!(walked_segments, BTreeSet::from(["cold", "hot", "other"]));
assert!(proposed.iter().all(|segment| walked_segments.contains(segment.as_str())));
assert_eq!(
walked_segments.len() - proposed.len(),
2,
"the two non-proposed segments must still be walked"
);
eprintln!(
"segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified",
paths.len()
);
} else {
assert!(proposed_walked.lock().expect("read disabled observations").is_empty());
}
// Compare semantic values because map encoding order is not content identity.
(paths, cache_value(&loaded))
}
#[tokio::test]
#[serial]
async fn segment_observation_on_off_preserves_actual_walk_and_saved_cache() {
let off = walk_and_save(false).await;
let on = walk_and_save(true).await;
assert_eq!(off.0, on.0, "diagnostics must not change actual traversal order or coverage");
assert_eq!(off.1, on.1, "diagnostics must not change the saved cache result");
}
@@ -0,0 +1,181 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use std::path::{Path, PathBuf};
use tokio::io::AsyncReadExt;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST";
struct Observation {
root: PathBuf,
limit: u64,
entries: u64,
name_bytes: u64,
}
static OBSERVATION: Mutex<Option<Observation>> = Mutex::new(None);
// Only the selected synthetic disk is observed; concurrent unrelated scanners
// do not consume its budget. This hook is absent from non-test builds.
pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) {
let mut guard = OBSERVATION.lock().expect("enumeration observation lock");
if let Some(observation) = guard.as_mut()
&& Path::new(dir).starts_with(&observation.root)
{
observation.entries += 1;
observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name");
if observation.entries >= observation.limit {
budget.cancel_for_runtime();
}
}
}
struct ObservationGuard;
impl Drop for ObservationGuard {
fn drop(&mut self) {
*OBSERVATION.lock().expect("enumeration observation cleanup") = None;
}
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct Request {
workspace: PathBuf,
objects: usize,
raw_entry_budget: u64,
round: u32,
}
async fn read_bounded(path: &Path) -> Vec<u8> {
let file = tokio::fs::File::open(path).await.expect("open fixture artifact");
let mut bytes = Vec::new();
file.take(MAX_CACHE_BYTES + 1)
.read_to_end(&mut bytes)
.await
.expect("read fixture artifact");
assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES);
bytes
}
async fn round(request: &Request) -> serde_json::Value {
assert!((1..=1024).contains(&request.objects));
assert!((1..=4096).contains(&request.raw_entry_budget));
assert!(request.round < 64);
let disk_root = request.workspace.join("disk");
let cache_path = request.workspace.join("cache.bin");
if request.round == 0 {
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
for index in 0..request.objects {
let object = format!("object-{index:04}");
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
let bytes = metadata_for_object_version("bucket", &object, Some(version));
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
}
let mut initial = DataUsageCache::default();
initial.info.name = "bucket".to_string();
initial.info.skip_healing = true;
initial.info.snapshot_complete = false;
initial.replace("bucket", "", DataUsageEntry::default());
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
.await
.expect("persist initial cache");
}
let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan");
assert_eq!(cache.info.name, "bucket");
let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects;
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open synthetic disk in this process");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
*OBSERVATION.lock().expect("install observation") = Some(Observation {
root: disk.path(),
limit: request.raw_entry_budget,
entries: 0,
name_bytes: 0,
});
let _observation_guard = ObservationGuard;
let result = scan_data_folder(
budget.token(),
budget.clone(),
vec![disk.clone()],
disk,
cache.clone(),
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let (returned, outcome) = match result {
Ok(cache) => (cache, "complete"),
Err(ScannerError::PartialCache(cache)) => (*cache, "partial"),
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {
(cache, "cancelled_without_cache")
}
Err(error) => panic!("unexpected real scanner failure: {error}"),
};
let encoded = returned.marshal_msg().expect("returned cache codec");
assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES);
tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache");
let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec");
let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root");
let scanned = returned.checked_flatten("bucket").expect("returned bucket root");
assert_eq!(
(retained.objects, retained.versions, retained.size),
(scanned.objects, scanned.versions, scanned.size)
);
assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete);
let guard = OBSERVATION.lock().expect("read observation");
let observation = guard.as_ref().expect("installed observation");
serde_json::json!({
"schema": 1, "pid": std::process::id(), "round": request.round,
"objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget,
"raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes,
"objects_processed": budget.progress().0,
"objects_before": before, "objects_retained": retained.objects,
"versions_retained": retained.versions, "bytes_retained": retained.size,
"snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome,
})
}
/// Default CI is a positive healthy control. The external driver selects the
/// same worker in a fresh OS process per round and applies its strict oracle.
#[tokio::test]
#[serial]
async fn enumeration_restart_worker() {
if let Some(path) = std::env::var_os(REQUEST_ENV) {
let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request");
let report = round(&request).await;
tokio::fs::write(
request.workspace.join(format!("round-{}.json", request.round)),
serde_json::to_vec(&report).expect("report JSON"),
)
.await
.expect("write worker report");
} else {
let temp = tempfile::tempdir().expect("healthy fixture directory");
let report = round(&Request {
workspace: temp.path().to_path_buf(),
objects: 4,
raw_entry_budget: 16,
round: 0,
})
.await;
assert_eq!(report["outcome"], "complete");
assert_eq!(report["snapshot_complete"], true);
assert_eq!(report["objects_retained"], 4);
assert_eq!(report["versions_retained"], 4);
assert_eq!(report["bytes_retained"], 4);
assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}");
}
}
+97 -2
View File
@@ -183,6 +183,18 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
return None;
}
// Completed maintenance also covers ordinary usage. Keep its exact stored
// proof for cache reuse, and reject mixtures of different set work proofs.
let baseline_plan_digest = DataUsageScanPlanDigest(baseline.usage_snapshot_set_states.first()?.scan_plan_digest?);
if ![
proof.scan_plan_digest,
scanner_bucket_work_digest(proof.scan_plan_digest, HealScanMode::Normal, true),
scanner_bucket_work_digest(proof.scan_plan_digest, HealScanMode::Deep, true),
]
.contains(&baseline_plan_digest)
{
return None;
}
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
for state in &baseline.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
@@ -192,13 +204,13 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
|| state.scan_plan_digest != Some(baseline_plan_digest.0)
{
return None;
}
}
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
(states == *proof.expected_sources).then_some(baseline_plan_digest)
}
fn scoped_scan_scope_from_dirty_buckets(
@@ -271,6 +283,9 @@ pub struct ScannerBucketScanPlan {
all_buckets: Arc<Vec<BucketInfo>>,
scope: ScannerBucketScanScope,
digest: DataUsageScanPlanDigest,
/// Includes mutation generations even when the set planner uses a structural digest.
bucket_coverage_digest: DataUsageScanPlanDigest,
requires_full_scan: bool,
// Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
execution_digest: DataUsageScanPlanDigest,
leader_epoch: u64,
@@ -314,6 +329,53 @@ fn scanner_bucket_plan_digest(buckets: &[BucketInfo], activity_digest: [u8; 32])
DataUsageScanPlanDigest(hasher.finalize().into())
}
fn scanner_bucket_inventory_is_complete(
all_buckets: &[BucketInfo],
buckets_by_source: &HashMap<DataUsageCacheSource, Vec<BucketInfo>>,
) -> bool {
let inventory = all_buckets
.iter()
.map(|bucket| (bucket.name.as_str(), bucket.created))
.collect::<HashMap<_, _>>();
if inventory.len() != all_buckets.len() || inventory.keys().any(|name| name.is_empty() || *name == DATA_USAGE_ROOT) {
return false;
}
let mut covered = HashSet::with_capacity(inventory.len());
for buckets in buckets_by_source.values() {
let mut set_names = HashSet::with_capacity(buckets.len());
for bucket in buckets {
if !set_names.insert(bucket.name.as_str()) || inventory.get(bucket.name.as_str()) != Some(&bucket.created) {
return false;
}
covered.insert(bucket.name.as_str());
}
}
covered.len() == inventory.len()
}
// Bind known work requirements before both local and remote cache admission.
// Matching requirements remain reusable for the same intent; this is not a
// new deadline or a durable generation for newly due maintenance.
fn scanner_bucket_work_digest(
scan_plan_digest: DataUsageScanPlanDigest,
scan_mode: HealScanMode,
requires_full_scan: bool,
) -> DataUsageScanPlanDigest {
if scan_mode == HealScanMode::Normal && !requires_full_scan {
return scan_plan_digest;
}
let mut hasher = Sha256::new();
hasher.update(b"scanner-bucket-work-v1");
hasher.update(scan_plan_digest.0);
hasher.update([match scan_mode {
HealScanMode::Unknown => 0,
HealScanMode::Normal => 1,
HealScanMode::Deep => 2,
}]);
hasher.update([u8::from(requires_full_scan || scan_mode == HealScanMode::Deep)]);
DataUsageScanPlanDigest(hasher.finalize().into())
}
fn scanner_bucket_cache_digest(
scan_plan_digest: DataUsageScanPlanDigest,
dirty_generation: Option<u64>,
@@ -711,6 +773,39 @@ pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>>
disks
}
pub(crate) async fn scanner_bucket_checkpoint_identity(
set: &SetDisks,
bucket: &str,
publication_epoch: u64,
tier_registry_generation: u64,
scan_mode: HealScanMode,
) -> Result<crate::DataUsageScanIdentity> {
let bucket_incarnation = set.bucket_incarnation_id_from_disk(bucket).await?;
let disks = set
.format
.erasure
.sets
.get(set.set_index)
.filter(|disks| !disks.is_empty())
.ok_or_else(|| Error::other("scanner checkpoint set layout is absent"))?;
if set.format.id.is_nil() || disks.iter().any(uuid::Uuid::is_nil) {
return Err(Error::other("scanner checkpoint set layout has a nil identity"));
}
let mut digest = Sha256::new();
digest.update(set.format.id.as_bytes());
for disk in disks {
digest.update(disk.as_bytes());
}
Ok(crate::DataUsageScanIdentity {
version: 1,
bucket_incarnation,
set_layout: crate::DataUsageScanPlanDigest(digest.finalize().into()),
publication_epoch,
tier_registry_generation,
scan_mode,
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScannerCycleDeferReason {
ActivityBaselineUnavailable,
+107 -27
View File
@@ -112,6 +112,10 @@ pub(crate) fn current_cache_root_entry_with_generation(
let metadata_is_current = cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_progress.is_none()
&& cache.info.scan_checkpoint.is_none()
&& cache.info.scan_resume_after.is_none()
&& cache.info.scan_coverage_receipt.is_none()
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
@@ -137,6 +141,7 @@ pub(crate) enum DataUsageCacheScanState {
pub(crate) struct DataUsageCacheReuseOptions {
pub(crate) require_source: bool,
pub(crate) tier_registry_generation: Option<u64>,
pub(crate) checkpoint_identity: Option<crate::DataUsageScanIdentity>,
}
#[cfg(test)]
@@ -159,6 +164,7 @@ pub(crate) fn current_cache_root_or_prepare(
DataUsageCacheReuseOptions {
require_source,
tier_registry_generation: None,
checkpoint_identity: None,
},
)
}
@@ -172,6 +178,12 @@ pub(crate) fn current_cache_root_or_prepare_with_generation(
scan_plan_digest: DataUsageScanPlanDigest,
options: DataUsageCacheReuseOptions,
) -> DataUsageCacheScanState {
if cache.info.next_cycle <= next_cycle
&& cache.info.leader_epoch <= leader_epoch
&& cache.info.scan_identity != options.checkpoint_identity
{
cache.info.scan_plan_digest = None;
}
if options.tier_registry_generation.is_some_and(|generation| {
cache.info.next_cycle <= next_cycle
&& cache.info.leader_epoch <= leader_epoch
@@ -193,7 +205,12 @@ pub(crate) fn current_cache_root_or_prepare_with_generation(
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
current => DataUsageCacheScanState::Prepared {
invalid_current: current.err(),
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, options.require_source),
outcome: match options.checkpoint_identity.filter(crate::DataUsageScanIdentity::is_valid) {
Some(identity) => {
cache.prepare_bucket_checkpoint(name, next_cycle, leader_epoch, source, scan_plan_digest, identity)
}
None => cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, options.require_source),
},
},
}
}
@@ -213,10 +230,87 @@ pub(super) fn cache_snapshot_is_current(
)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct ScannerSnapshotIdentity {
pub(super) cycle: u64,
pub(super) leader_epoch: u64,
pub(super) plan_digest: DataUsageScanPlanDigest,
pub(super) coverage_digest: DataUsageScanPlanDigest,
pub(super) tier_registry_generation: Option<u64>,
}
pub(super) struct ScannerSnapshotScope<'a> {
pub(super) sources: &'a HashSet<DataUsageCacheSource>,
pub(super) buckets: &'a [String],
pub(super) identity: ScannerSnapshotIdentity,
}
#[derive(Debug, PartialEq, Eq, thiserror::Error)]
pub(super) enum ScannerSnapshotValidationError {
#[error("scanner snapshot does not cover the expected complete sets")]
IncompleteSets,
#[error("scanner snapshot does not match the requested generation")]
GenerationMismatch,
#[error("scanner snapshot bucket inventory is invalid")]
InvalidInventory,
#[error("scanner snapshot root is incomplete or corrupt")]
InvalidRoot,
}
struct ValidatedScannerSnapshot<'a> {
results: &'a [DataUsageCache],
last_update: SystemTime,
}
impl<'a> ValidatedScannerSnapshot<'a> {
fn validate(
results: &'a [DataUsageCache],
scope: &ScannerSnapshotScope<'_>,
) -> std::result::Result<Self, ScannerSnapshotValidationError> {
if !scanner_results_form_complete_snapshot(results, scope.sources) {
return Err(ScannerSnapshotValidationError::IncompleteSets);
}
let bucket_keys = scope
.buckets
.iter()
.map(|bucket| crate::hash_path(bucket).key())
.collect::<HashSet<_>>();
if bucket_keys.len() != scope.buckets.len()
|| scope
.buckets
.iter()
.any(|bucket| bucket.is_empty() || bucket == DATA_USAGE_ROOT)
{
return Err(ScannerSnapshotValidationError::InvalidInventory);
}
for result in results {
if result.info.next_cycle != scope.identity.cycle
|| result.info.leader_epoch != scope.identity.leader_epoch
|| result.info.scan_plan_digest != Some(scope.identity.plan_digest)
|| result.info.scan_coverage_digest != Some(scope.identity.coverage_digest)
|| result.info.tier_registry_generation != scope.identity.tier_registry_generation
{
return Err(ScannerSnapshotValidationError::GenerationMismatch);
}
if result.info.name != DATA_USAGE_ROOT
|| result.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
|| !result.has_complete_root_inventory(&bucket_keys)
{
return Err(ScannerSnapshotValidationError::InvalidRoot);
}
}
let last_update = results
.iter()
.filter_map(|result| result.info.last_update)
.max()
.ok_or(ScannerSnapshotValidationError::IncompleteSets)?;
Ok(Self { results, last_update })
}
}
pub(super) fn completed_data_usage_info(
results: &[DataUsageCache],
expected_sources: &HashSet<DataUsageCacheSource>,
all_buckets: &[String],
scope: &ScannerSnapshotScope<'_>,
tier_registry_names: &[String],
bucket_plan_complete: bool,
budget_elapsed: bool,
@@ -229,26 +323,10 @@ pub(super) fn completed_data_usage_info(
if !should_publish_completed_snapshot(completed_set_count, results.len(), budget_elapsed, cancelled) {
return None;
}
if !scanner_results_form_complete_snapshot(results, expected_sources) {
return None;
}
// A generation is comparable across nodes because it is derived from the
// frozen registry names. Cycle and leader fencing remain separate cache
// metadata. Legacy peers omit the generation; an all-legacy result remains
// readable, but mixing legacy and new (or two new generations) would make
// the per-tier accounting ambiguous.
let registry_generation = results.first()?.info.tier_registry_generation;
if results.iter().any(|result| match registry_generation {
Some(generation) => result.info.tier_registry_generation != Some(generation),
None => result.info.tier_registry_generation.is_some(),
}) {
return None;
}
if results.iter().any(|result| result.root().is_none()) {
return None;
}
let validated = ValidatedScannerSnapshot::validate(results, scope).ok()?;
let results = validated.results;
let all_buckets = scope.buckets;
let registry_generation = scope.identity.tier_registry_generation;
let mut total = DataUsageEntry::default();
let mut bucket_entries = HashMap::with_capacity(all_buckets.len());
@@ -273,7 +351,7 @@ pub(super) fn completed_data_usage_info(
return None;
}
let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?;
let merged_last_update = validated.last_update;
let buckets_usage = bucket_entries
.iter()
.map(|(bucket, entry)| Some((bucket.clone(), checked_bucket_usage_info(entry)?)))
@@ -300,8 +378,8 @@ pub(super) fn completed_data_usage_info(
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
scanner_epoch: Some(results.first()?.info.leader_epoch),
scanner_cycle: Some(scope.identity.cycle),
scanner_epoch: Some(scope.identity.leader_epoch),
objects_total_count: u64::try_from(total.objects).ok()?,
versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
@@ -609,6 +687,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
expected_publication_epoch: u64,
) -> Option<SystemTime> {
let source = cache_snapshot.info.source?;
let coverage_digest = cache_snapshot.info.scan_coverage_digest?;
let execution_digest = cache_snapshot.info.scan_execution_digest?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
Ok(guard) => guard,
@@ -674,7 +753,8 @@ pub(super) async fn persist_and_publish_cache_snapshot(
);
return None;
}
if persisted.info.scan_execution_digest == Some(execution_digest)
if persisted.info.scan_coverage_digest == Some(coverage_digest)
&& persisted.info.scan_execution_digest == Some(execution_digest)
&& matches!(
current_cache_root_entry_with_generation(
&persisted,
+44 -13
View File
@@ -39,6 +39,12 @@ pub(super) fn prepare_scoped_set_scan(
else {
return None;
};
// The existing cache does not bind each bucket to a durable incarnation.
// Listing creation times can come from volume metadata, so even Some(time)
// cannot prove that an unselected same-name bucket is the cached bucket.
if all_buckets.iter().any(|bucket| !selected_buckets.contains(&bucket.name)) {
return None;
}
if selected_buckets.is_empty()
|| !old_cache.info.snapshot_complete
|| old_cache.info.last_update.is_none()
@@ -49,7 +55,7 @@ pub(super) fn prepare_scoped_set_scan(
|| old_cache.info.source != Some(generation.source)
|| old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest)
|| old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
|| old_cache.checked_flatten_complete_scope(DATA_USAGE_ROOT).is_none()
|| !old_cache.has_complete_root_inventory(&old_cache.find(DATA_USAGE_ROOT)?.children)
{
return None;
}
@@ -74,21 +80,12 @@ pub(super) fn prepare_scoped_set_scan(
cache: HashMap::new(),
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let root_hash = crate::hash_path(DATA_USAGE_ROOT);
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
}
if selected_buckets.contains(&bucket.name) {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
continue;
}
let bucket_hash = crate::hash_path(&bucket.name);
old_cache.find(&bucket.name)?;
cache.copy_with_children(old_cache, &bucket_hash, &Some(root_hash.clone()));
cache.find(&bucket.name)?;
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
Some(PreparedScopedSetScan {
@@ -118,6 +115,8 @@ impl ScannerIOCache for SetDisks {
all_buckets,
scope,
digest: scan_plan_digest,
bucket_coverage_digest,
requires_full_scan,
execution_digest,
leader_epoch,
tier_registry_generation,
@@ -127,6 +126,8 @@ impl ScannerIOCache for SetDisks {
pending_maintenance_work,
cache_cycle_floor,
} = scan_plan;
let scan_plan_digest = scanner_bucket_work_digest(scan_plan_digest, scan_mode, requires_full_scan);
let bucket_work_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
let pool_label = self.pool_index.to_string();
let set_label = self.set_index.to_string();
@@ -169,8 +170,9 @@ impl ScannerIOCache for SetDisks {
scan_plan_digest,
},
);
let mut scoped_cache = scoped_scan.map(|prepared| {
let mut scoped_cache = scoped_scan.map(|mut prepared| {
buckets = prepared.buckets;
prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest);
prepared.cache
});
if buckets.is_empty() {
@@ -186,6 +188,7 @@ impl ScannerIOCache for SetDisks {
tier_registry_generation: Some(tier_registry_generation),
source: Some(source),
scan_plan_digest: Some(scan_plan_digest),
scan_coverage_digest: Some(bucket_coverage_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
@@ -475,6 +478,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
scan_coverage_digest: Some(bucket_coverage_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
lkg_snapshot_complete: old_cache.info.lkg_snapshot_complete,
lkg_next_cycle: old_cache.info.lkg_next_cycle,
@@ -644,7 +648,7 @@ impl ScannerIOCache for SetDisks {
let cache_name = path_join_buf(&[&bucket.name, DATA_USAGE_CACHE_NAME]);
let bucket_scan_plan_digest =
scanner_bucket_cache_digest(execution_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
scanner_bucket_cache_digest(bucket_work_digest, dirty_usage_buckets_clone.get(&bucket.name).copied());
if let Some(server_epoch) = remote_server_epoch {
let request_sequence = remote_session_sequence;
@@ -890,6 +894,17 @@ impl ScannerIOCache for SetDisks {
continue;
}
};
// Lack of an authoritative legacy identity disables the new
// checkpoint protocol; the existing full rebuild remains available.
let checkpoint_identity = scanner_bucket_checkpoint_identity(
&store_clone_clone,
&bucket.name,
expected_publication_epoch_clone,
tier_registry_generation,
scan_mode,
)
.await
.ok();
let scan_state = current_cache_root_or_prepare_with_generation(
&mut cache,
&bucket.name,
@@ -900,6 +915,7 @@ impl ScannerIOCache for SetDisks {
DataUsageCacheReuseOptions {
require_source: require_cache_source,
tier_registry_generation: Some(tier_registry_generation),
checkpoint_identity,
},
);
let outcome = match scan_state {
@@ -1049,6 +1065,21 @@ impl ScannerIOCache for SetDisks {
}
}
};
if let Some(expected) = checkpoint_identity
&& scanner_bucket_checkpoint_identity(
&store_clone_clone,
&bucket.name,
expected_publication_epoch_clone,
tier_registry_generation,
scan_mode,
)
.await
.ok()
!= Some(expected)
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
continue;
}
let scan_outcome = match scan_result {
Ok(scan_outcome) => scan_outcome,
Err(e) => {
+40 -10
View File
@@ -72,6 +72,9 @@ where
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
requires_full_scan: true,
#[cfg(test)]
resolved_scope_observer: None,
};
nsscanner_with_storage_status_scoped(store, request).await
}
@@ -85,6 +88,10 @@ pub(crate) struct ScannerCycleRequest {
pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope,
pub(crate) persisted_usage_baseline: Option<Bytes>,
/// Scheduled maintenance must visit clean buckets even with a valid dirty scope.
pub(crate) requires_full_scan: bool,
#[cfg(test)]
pub(crate) resolved_scope_observer: Option<tokio::sync::oneshot::Sender<ScannerBucketScanScope>>,
}
struct ScannerBucketScopeResolution<'a> {
@@ -93,6 +100,7 @@ struct ScannerBucketScopeResolution<'a> {
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
all_buckets: &'a [BucketInfo],
requires_full_scan: bool,
}
async fn resolve_scanner_bucket_scan_scope<S>(
@@ -103,6 +111,9 @@ async fn resolve_scanner_bucket_scan_scope<S>(
where
S: ScannerStorage,
{
if resolution.requires_full_scan {
return ScannerBucketScanScope::default();
}
if !resolution.requested_scope.is_default()
|| !resolution.dirty_usage_snapshot.covers_all_pending
|| resolution.dirty_usage_snapshot.generation == u64::MAX
@@ -172,6 +183,9 @@ where
scan_mode,
scan_scope,
persisted_usage_baseline,
requires_full_scan,
#[cfg(test)]
resolved_scope_observer,
} = request;
let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -260,13 +274,13 @@ where
}
}
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
let scan_plan_digest =
bucket_plan_complete &= scanner_bucket_inventory_is_complete(&all_buckets, &buckets_by_source);
let structural_scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
let mut execution_hasher = Sha256::new();
execution_hasher.update(scan_plan_digest.0);
execution_hasher.update(activity_digest);
let execution_digest = DataUsageScanPlanDigest(execution_hasher.finalize().into());
let scan_plan_digest = scanner_bucket_work_digest(structural_scan_plan_digest, scan_mode, requires_full_scan);
let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
store,
@@ -278,14 +292,19 @@ where
expected_sources: &expected_sources,
leader_epoch,
want_cycle,
scan_plan_digest,
scan_plan_digest: structural_scan_plan_digest,
},
activity_before: &activity_before,
dirty_usage_snapshot: &dirty_usage_snapshot,
all_buckets: &all_buckets,
requires_full_scan: requires_full_scan || scan_mode == HealScanMode::Deep,
},
)
.await;
#[cfg(test)]
if let Some(observer) = resolved_scope_observer {
let _ = observer.send(scan_scope.clone());
}
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
let tier_registry_generation = tier_registry.generation;
@@ -415,7 +434,9 @@ where
buckets: set_buckets,
all_buckets: Arc::clone(&all_buckets),
scope: scan_scope.clone(),
digest: scan_plan_digest,
digest: structural_scan_plan_digest,
bucket_coverage_digest,
requires_full_scan,
execution_digest,
leader_epoch,
tier_registry_generation,
@@ -543,8 +564,17 @@ where
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
let completed_usage = completed_data_usage_info(
&results,
&expected_sources,
&all_bucket_names,
&ScannerSnapshotScope {
sources: &expected_sources,
buckets: &all_bucket_names,
identity: ScannerSnapshotIdentity {
cycle: want_cycle,
leader_epoch,
plan_digest: scan_plan_digest,
coverage_digest: bucket_coverage_digest,
tier_registry_generation: Some(tier_registry_generation),
},
},
&tier_registry.names,
bucket_plan_complete,
budget_elapsed,
@@ -17,6 +17,33 @@ use crate::data_usage_define::{UNKNOWN_TIER, UnknownTierStats, hash_path};
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage, TierAccountingProof};
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
const TEST_COVERAGE_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([6; 32]);
#[test]
fn scanner_bucket_inventory_requires_exact_unique_set_union() {
let first = BucketInfo {
name: "first".to_string(),
..Default::default()
};
let second = BucketInfo {
name: "second".to_string(),
..Default::default()
};
let source = DataUsageCacheSource::new(0, 0);
let mut sets = HashMap::from([(source, vec![first.clone()])]);
assert!(scanner_bucket_inventory_is_complete(std::slice::from_ref(&first), &sets));
assert!(!scanner_bucket_inventory_is_complete(&[first.clone(), second.clone()], &sets));
assert!(!scanner_bucket_inventory_is_complete(&[], &sets));
assert!(!scanner_bucket_inventory_is_complete(&[first.clone(), first.clone()], &sets));
sets.insert(source, vec![first.clone(), first.clone()]);
assert!(!scanner_bucket_inventory_is_complete(std::slice::from_ref(&first), &sets));
sets.insert(source, vec![second]);
assert!(!scanner_bucket_inventory_is_complete(std::slice::from_ref(&first), &sets));
let mut recreated = first.clone();
recreated.created = Some(OffsetDateTime::UNIX_EPOCH);
sets.insert(source, vec![recreated]);
assert!(!scanner_bucket_inventory_is_complete(&[first], &sets));
}
#[test]
fn should_publish_completed_snapshot_requires_full_clean_cycle() {
@@ -79,6 +106,7 @@ fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64, source:
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
scan_coverage_digest: Some(TEST_COVERAGE_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
@@ -108,7 +136,139 @@ fn completed_data_usage_info_for_test(
cancelled: bool,
) -> Option<(DataUsageInfo, SystemTime)> {
let expected_sources = results.iter().filter_map(|result| result.info.source).collect::<HashSet<_>>();
completed_data_usage_info(results, &expected_sources, all_buckets, &[], true, budget_elapsed, cancelled)
completed_usage_for_scope(results, &expected_sources, all_buckets, &[], true, budget_elapsed, cancelled)
}
fn completed_usage_for_scope(
results: &[DataUsageCache],
expected_sources: &HashSet<DataUsageCacheSource>,
all_buckets: &[String],
tier_registry_names: &[String],
bucket_plan_complete: bool,
budget_elapsed: bool,
cancelled: bool,
) -> Option<(DataUsageInfo, SystemTime)> {
let first = results.first()?;
completed_data_usage_info(
results,
&ScannerSnapshotScope {
sources: expected_sources,
buckets: all_buckets,
identity: ScannerSnapshotIdentity {
cycle: first.info.next_cycle,
leader_epoch: first.info.leader_epoch,
plan_digest: TEST_PLAN_DIGEST,
coverage_digest: TEST_COVERAGE_DIGEST,
tier_registry_generation: first.info.tier_registry_generation,
},
},
tier_registry_names,
bucket_plan_complete,
budget_elapsed,
cancelled,
)
}
#[test]
fn completed_data_usage_info_rejects_duplicate_bucket_inventory() {
let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let buckets = vec!["bucket".to_string(), "bucket".to_string()];
assert!(completed_data_usage_info_for_test(&[set], &buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_extra_or_detached_bucket_data() {
let buckets = vec!["bucket".to_string()];
let mut set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
set.replace(
"unlisted",
DATA_USAGE_ROOT,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(completed_data_usage_info_for_test(&[set.clone()], &buckets, false, false).is_none());
set.cache
.get_mut(DATA_USAGE_ROOT)
.expect("set root")
.children
.remove(&hash_path("unlisted").key());
assert!(
completed_data_usage_info_for_test(&[set], &buckets, false, false).is_none(),
"orphaned data must not disappear from authoritative accounting"
);
}
#[test]
fn completed_data_usage_info_rejects_disconnected_expected_bucket() {
let buckets = vec!["bucket".to_string()];
let mut set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
set.cache.get_mut(DATA_USAGE_ROOT).expect("set root").children.clear();
assert!(completed_data_usage_info_for_test(&[set], &buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_rejects_root_scalar_data_and_unknown_key_format() {
let buckets = vec!["bucket".to_string()];
let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut scalar_root = set.clone();
scalar_root.cache.get_mut(DATA_USAGE_ROOT).expect("set root").size = 10;
assert!(completed_data_usage_info_for_test(&[scalar_root], &buckets, false, false).is_none());
let mut future_format = set;
future_format.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT + 1;
assert!(completed_data_usage_info_for_test(&[future_format], &buckets, false, false).is_none());
}
#[test]
fn completed_data_usage_info_binds_all_results_to_requested_identity() {
let buckets = vec!["bucket".to_string()];
let source = DataUsageCacheSource::new(0, 0);
let sources = HashSet::from([source]);
let set = completed_root_cache("bucket", 2, 10, source);
let identity = ScannerSnapshotIdentity {
cycle: 0,
leader_epoch: 0,
plan_digest: TEST_PLAN_DIGEST,
coverage_digest: TEST_COVERAGE_DIGEST,
tier_registry_generation: None,
};
let results = [set];
for expected in [
ScannerSnapshotIdentity { cycle: 1, ..identity },
ScannerSnapshotIdentity {
leader_epoch: 1,
..identity
},
ScannerSnapshotIdentity {
plan_digest: DataUsageScanPlanDigest([9; 32]),
..identity
},
ScannerSnapshotIdentity {
tier_registry_generation: Some(1),
..identity
},
ScannerSnapshotIdentity {
coverage_digest: DataUsageScanPlanDigest([4; 32]),
..identity
},
] {
let scope = ScannerSnapshotScope {
sources: &sources,
buckets: &buckets,
identity: expected,
};
assert!(completed_data_usage_info(&results, &scope, &[], true, false, false).is_none());
}
let scope = ScannerSnapshotScope {
sources: &sources,
buckets: &buckets,
identity,
};
let (usage, _) = completed_data_usage_info(&results, &scope, &[], true, false, false)
.expect("the requested complete scope remains publishable");
assert_eq!(usage.objects_total_count, 2);
assert!(usage.is_complete_bucket_usage_snapshot());
}
fn lkg_root_cache(bucket: &str, objects: usize, source: DataUsageCacheSource) -> DataUsageCache {
@@ -136,7 +296,7 @@ fn partial_usage_is_observational_not_authoritative_for_quota() {
let expected = HashSet::from([current_source, stalled_source]);
assert!(
completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, &[], true, false, false)
completed_usage_for_scope(&[current.clone(), stalled.clone()], &expected, &all_buckets, &[], true, false, false)
.is_none()
);
let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3)
@@ -509,7 +669,7 @@ fn completed_data_usage_info_accepts_unknown_only_with_current_registry_generati
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]);
assert!(
completed_data_usage_info(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_some()
completed_usage_for_scope(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_some()
);
}
@@ -547,7 +707,7 @@ fn completed_data_usage_info_rejects_non_registry_tier_in_current_generation() {
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]);
assert!(
completed_data_usage_info(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_none()
completed_usage_for_scope(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_none()
);
}
@@ -619,6 +779,7 @@ fn current_cache_root_with_new_tier_generation_resets_old_cache() {
DataUsageCacheReuseOptions {
require_source: false,
tier_registry_generation: Some(2),
checkpoint_identity: None,
},
);
@@ -698,6 +859,8 @@ fn completed_data_usage_info_publishes_confirmed_empty_namespace() {
source: Some(DataUsageCacheSource::new(0, 0)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
scan_coverage_digest: Some(TEST_COVERAGE_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -855,7 +1018,7 @@ fn completed_data_usage_info_requires_exact_topology_sources() {
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]);
assert!(
completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, &[], true, false, false)
completed_usage_for_scope(&[first_set, unexpected_set], &expected_sources, &all_buckets, &[], true, false, false)
.is_none()
);
}
@@ -866,7 +1029,7 @@ fn completed_data_usage_info_rejects_incomplete_bucket_plan() {
let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]);
assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, &[], false, false, false).is_none());
assert!(completed_usage_for_scope(&[set], &expected_sources, &all_buckets, &[], false, false, false).is_none());
}
#[test]
@@ -1176,6 +1339,90 @@ fn dirty_bucket_cache_digest_changes_with_generation() {
assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second));
}
#[test]
fn scoped_scan_bucket_work_proof_fences_same_cycle_cache() {
let source = DataUsageCacheSource::new(0, 0);
let structural_plan = DataUsageScanPlanDigest([9; 32]);
let normal_plan = scanner_bucket_work_digest(structural_plan, HealScanMode::Normal, false);
assert_eq!(normal_plan, structural_plan, "ordinary work keeps the existing digest contract");
for (scan_mode, full) in [(HealScanMode::Deep, false), (HealScanMode::Normal, true)] {
let requested_plan = scanner_bucket_work_digest(structural_plan, scan_mode, full);
assert_ne!(requested_plan, normal_plan);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "cold".to_string(),
next_cycle: 7,
leader_epoch: 11,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(normal_plan),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.replace("cold", "", DataUsageEntry::default());
assert!(cache_snapshot_is_current(&cache, "cold", source, 7, 11, normal_plan));
assert!(matches!(
current_cache_root_or_prepare(&mut cache, "cold", source, 7, 11, requested_plan, true),
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
..
}
));
assert!(cache.cache.is_empty(), "different work requirements must enter a fresh walk");
cache.replace("cold", "", DataUsageEntry::default());
cache.info.snapshot_complete = true;
cache.info.last_update = Some(SystemTime::UNIX_EPOCH);
assert!(
matches!(
current_cache_root_or_prepare(&mut cache, "cold", source, 7, 11, requested_plan, true),
DataUsageCacheScanState::Current(_)
),
"completed matching work may satisfy the same intent retry"
);
}
assert_eq!(
scanner_bucket_work_digest(structural_plan, HealScanMode::Deep, false),
scanner_bucket_work_digest(structural_plan, HealScanMode::Deep, true)
);
}
#[test]
fn scoped_scan_complete_root_requires_current_coverage_from_every_set() {
let sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]);
let buckets = vec!["bucket".to_string()];
let coverage = DataUsageScanPlanDigest([4; 32]);
let scope = ScannerSnapshotScope {
sources: &sources,
buckets: &buckets,
identity: ScannerSnapshotIdentity {
cycle: 0,
leader_epoch: 0,
plan_digest: TEST_PLAN_DIGEST,
coverage_digest: coverage,
tier_registry_generation: None,
},
};
for (first_coverage, second_coverage, valid) in [
(Some(coverage), Some(coverage), true),
(None, Some(coverage), false),
(Some(coverage), None, false),
(None, None, false),
(Some(coverage), Some(DataUsageScanPlanDigest([5; 32])), false),
] {
let mut first = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
let mut second = completed_root_cache("bucket", 3, 10, DataUsageCacheSource::new(1, 0));
first.info.scan_coverage_digest = first_coverage;
second.info.scan_coverage_digest = second_coverage;
assert_eq!(
completed_data_usage_info(&[first, second], &scope, &[], true, false, false).is_some(),
valid
);
}
}
#[test]
fn scanner_cache_lock_resource_is_scoped_to_cache_source() {
let cache_name = "photos/.usage-cache.bin";
+348 -8
View File
@@ -124,6 +124,51 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
(temp_dir, store)
}
async fn wait_for_namespace_commit_tails(store: &ECStore) {
tokio::time::timeout(Duration::from_secs(30), async {
while store.scanner_data_usage_publication_blocked().await {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("namespace commit tails should drain before the scanner fixture runs");
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_bucket_identity_uses_its_set_instance_owner() {
let (_first_dir, first) = setup_two_pool_scanner_store().await;
first
.make_bucket("checkpoint-identity", &MakeBucketOptions::default())
.await
.expect("first instance bucket");
let first_identity =
scanner_bucket_checkpoint_identity(&first.pools[0].disk_set[0], "checkpoint-identity", 0, 7, HealScanMode::Normal)
.await
.expect("first durable identity");
let (_second_dir, second) = setup_two_pool_scanner_store().await;
second
.make_bucket("checkpoint-identity", &MakeBucketOptions::default())
.await
.expect("second instance bucket");
let second_identity =
scanner_bucket_checkpoint_identity(&second.pools[0].disk_set[0], "checkpoint-identity", 0, 7, HealScanMode::Normal)
.await
.expect("second durable identity");
assert_ne!(first_identity.bucket_incarnation, second_identity.bucket_incarnation);
assert_eq!(
scanner_bucket_checkpoint_identity(&first.pools[0].disk_set[0], "checkpoint-identity", 0, 7, HealScanMode::Normal)
.await
.expect("first owner remains bound"),
first_identity
);
assert!(
scanner_bucket_checkpoint_identity(&first.pools[0].disk_set[0], "missing-checkpoint-bucket", 0, 7, HealScanMode::Normal)
.await
.is_err()
);
}
#[tokio::test]
#[serial]
async fn scanner_cache_locks_block_same_source_workers() {
@@ -283,6 +328,213 @@ async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
}
}
#[tokio::test]
#[serial]
async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
for bucket in ["hot-bucket", "cold-bucket"] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = ScannerPutObjReader::from_vec(b"initial".to_vec());
store.pools[0].disk_set[0]
.put_object(bucket, "initial", &mut reader, &ScannerObjectOptions::default())
.await
.expect("initial object should persist");
}
wait_for_namespace_commit_tails(store.as_ref()).await;
let mut baseline = None;
for (index, (scan_mode, requires_full_scan, explicit_scope)) in [
(HealScanMode::Normal, true, false),
(HealScanMode::Normal, false, false),
(HealScanMode::Deep, false, false),
(HealScanMode::Normal, true, false),
(HealScanMode::Deep, false, true),
(HealScanMode::Normal, true, true),
]
.into_iter()
.enumerate()
{
if index > 0 {
let mut reader = ScannerPutObjReader::from_vec(b"maintenance".to_vec());
store.pools[0].disk_set[0]
.put_object("cold-bucket", &format!("added-{index}"), &mut reader, &ScannerObjectOptions::default())
.await
.expect("cold bucket mutation should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
// Only the hot bucket is in the usage hint. The cold result must
// come from this cycle's storage walk, not its previous baseline.
record_dirty_usage_bucket("hot-bucket");
}
let requested_scope = if explicit_scope {
ScannerBucketScanScope::from_dirty_buckets(
HashSet::from(["hot-bucket".to_string()]),
DataUsageScanPlanDigest([7; 32]),
)
} else {
ScannerBucketScanScope::default()
};
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let (observer, observed_scope) = tokio::sync::oneshot::channel();
let cycle = u64::try_from(index + 1).expect("test cycle should fit");
let result = tokio::time::timeout(
Duration::from_secs(30),
nsscanner_with_storage_status_scoped(
store.as_ref(),
ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle: cycle,
leader_epoch: 11,
scan_mode,
scan_scope: requested_scope,
persisted_usage_baseline: baseline,
requires_full_scan,
resolved_scope_observer: Some(observer),
},
),
)
.await
.expect("cycle should finish within the test deadline")
.expect("cycle should succeed");
assert_eq!(result.status, ScannerCycleStatus::Complete, "cycle {cycle}");
let resolved = observed_scope.await.expect("production resolver should report its scope");
if index == 1 {
assert_eq!(
resolved.selected_buckets.as_deref(),
Some(&HashSet::from(["hot-bucket".to_string()])),
"ordinary dirty work must retain the existing planner"
);
} else {
assert!(resolved.is_default(), "cycle {cycle} must visit the full maintenance scope");
}
let mut snapshot = receiver.recv().await.expect("cycle should publish a snapshot");
assert!(snapshot.usage_snapshot_complete, "cycle {cycle}");
assert_eq!(
snapshot.buckets_usage["cold-bucket"].objects_count,
u64::try_from(index + 1).expect("count should fit")
);
assert_eq!(snapshot.buckets_usage["hot-bucket"].objects_count, 1);
assert_eq!(snapshot.scanner_cycle, Some(cycle));
assert_eq!(snapshot.scanner_epoch, Some(11));
snapshot.usage_snapshot_converged = Some(true);
baseline = Some(Bytes::from(serde_json::to_vec(&snapshot).expect("complete baseline should encode")));
}
clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn scoped_scan_same_cycle_maintenance_rewalks_after_root_delivery_failure() {
for (scan_mode, requires_full_scan) in [
(HealScanMode::Normal, false),
(HealScanMode::Deep, false),
(HealScanMode::Normal, true),
] {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
clear_dirty_usage_buckets_for_tests();
for bucket in ["hot-bucket", "cold-bucket"] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = ScannerPutObjReader::from_vec(b"initial".to_vec());
store.pools[0].disk_set[0]
.put_object(bucket, "initial", &mut reader, &ScannerObjectOptions::default())
.await
.expect("initial object should persist");
}
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, receiver) = mpsc::channel(1);
drop(receiver);
let failed = tokio::time::timeout(
Duration::from_secs(30),
nsscanner_with_storage_status_scoped(
store.as_ref(),
ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle: 7,
leader_epoch: 11,
scan_mode: HealScanMode::Normal,
scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
requires_full_scan: false,
resolved_scope_observer: None,
},
),
)
.await
.expect("normal scan should finish")
.expect_err("root delivery must fail after bucket cache persistence");
assert!(failed.to_string().contains("receiver closed"), "{failed}");
let cache_name = path_join_buf(&["cold-bucket", DATA_USAGE_CACHE_NAME]);
let mut cached = DataUsageCache::default();
cached
.load(store.pools[0].disk_set[0].clone(), &cache_name)
.await
.expect("normal bucket cache should have committed");
assert!(cached.info.snapshot_complete);
assert_eq!(cached.info.next_cycle, 7);
assert_eq!(
cached
.checked_flatten("cold-bucket")
.expect("cached root should be valid")
.objects,
1
);
let mut reader = ScannerPutObjReader::from_vec(b"maintenance".to_vec());
store.pools[0].disk_set[0]
.put_object("cold-bucket", "new", &mut reader, &ScannerObjectOptions::default())
.await
.expect("new cold object should persist");
record_dirty_usage_bucket("hot-bucket");
if scan_mode == HealScanMode::Normal && !requires_full_scan {
record_dirty_usage_bucket("cold-bucket");
}
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(30),
nsscanner_with_storage_status_scoped(
store.as_ref(),
ScannerCycleRequest {
ctx,
budget,
updates,
want_cycle: 7,
leader_epoch: 11,
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
requires_full_scan,
resolved_scope_observer: None,
},
),
)
.await
.expect("maintenance scan should finish")
.expect("maintenance scan should succeed");
assert_eq!(result.status, ScannerCycleStatus::Complete);
let snapshot = receiver.recv().await.expect("maintenance snapshot should be published");
assert_eq!(snapshot.scanner_cycle, Some(7));
assert_eq!(
snapshot.buckets_usage["cold-bucket"].objects_count, 2,
"{scan_mode:?}/full={requires_full_scan} must not replay the same-cycle Normal root"
);
clear_dirty_usage_buckets_for_tests();
}
}
#[tokio::test]
async fn data_usage_publish_fails_when_receiver_is_closed() {
let (updates, receiver) = mpsc::channel(1);
@@ -893,6 +1145,7 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
scan_coverage_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
tier_registry_generation: Some(13),
..Default::default()
@@ -1010,6 +1263,8 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
all_buckets: Arc::new(Vec::new()),
scope: ScannerBucketScanScope::default(),
digest: DataUsageScanPlanDigest([6; 32]),
bucket_coverage_digest: DataUsageScanPlanDigest([6; 32]),
requires_full_scan: false,
execution_digest: empty_execution,
leader_epoch: 11,
tier_registry_generation: 13,
@@ -1139,6 +1394,36 @@ fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
}
#[test]
fn scoped_scan_baseline_work_proof_requires_uniform_known_set_identity() {
let source = DataUsageCacheSource::new(1, 2);
let second_source = DataUsageCacheSource::new(1, 3);
let sources = HashSet::from([source, second_source]);
let structural = DataUsageScanPlanDigest([9; 32]);
let full = scanner_bucket_work_digest(structural, HealScanMode::Normal, true);
let deep = scanner_bucket_work_digest(structural, HealScanMode::Deep, true);
let encoded = complete_usage_baseline(source, full, 7, 11);
let baseline: DataUsageInfo = serde_json::from_slice(&encoded).expect("baseline should decode");
for (second_plan, expected) in [(full, Some(full)), (deep, None), (DataUsageScanPlanDigest([8; 32]), None)] {
let mut candidate = baseline.clone();
let mut second = candidate.usage_snapshot_set_states[0].clone();
second.set_index = 3;
second.scan_plan_digest = Some(second_plan.0);
candidate.usage_snapshot_set_states.push(second);
let data = Bytes::from(serde_json::to_vec(&candidate).expect("candidate should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&data),
expected_sources: &sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: structural,
}),
expected
);
}
}
fn peer_dirty_usage_snapshot(
instance_id: &str,
generation: u64,
@@ -1221,8 +1506,15 @@ fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state()
}
}
fn bucket_info_with_created_time(name: &str) -> BucketInfo {
BucketInfo {
created: Some(time::OffsetDateTime::UNIX_EPOCH),
..bucket_info(name)
}
}
#[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let current_digest = DataUsageScanPlanDigest([2; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20), ("deleted", 30)], baseline_digest);
@@ -1235,8 +1527,11 @@ fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
..Default::default()
},
);
let all_buckets = vec![bucket_info("stable"), bucket_info("dirty")];
let selected_buckets = Arc::new(HashSet::from(["dirty".to_string(), "deleted".to_string()]));
let all_buckets = vec![
bucket_info_with_created_time("stable"),
bucket_info_with_created_time("dirty"),
];
let selected_buckets = Arc::new(HashSet::from(["stable".to_string(), "dirty".to_string(), "deleted".to_string()]));
let prepared = prepare_scoped_set_scan(
&old_cache,
@@ -1256,12 +1551,16 @@ fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
)
.expect("complete matching set cache should support a scoped scan");
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
assert_eq!(
prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(),
["stable", "dirty"]
);
let stable = prepared
.cache
.checked_flatten("stable")
.expect("unselected bucket subtree should be retained");
assert_eq!((stable.size, stable.objects), (15, 2));
.expect("selected bucket placeholder should exist");
assert_eq!((stable.size, stable.objects), (0, 0));
assert!(prepared.cache.find("stable/prefix").is_none());
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
assert_eq!(prepared.cache.info.scan_plan_digest, Some(current_digest));
@@ -1272,11 +1571,41 @@ fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn scoped_set_scan_rejects_unbound_bucket_incarnations() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([2; 32]),
};
for created in [
None,
Some(OffsetDateTime::UNIX_EPOCH),
Some(OffsetDateTime::UNIX_EPOCH + time::Duration::days(1)),
] {
let mut stable = bucket_info("stable");
stable.created = created;
let buckets = vec![stable, bucket_info_with_created_time("dirty")];
assert!(
prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation).is_none(),
"missing identity, volume timestamps and same-name recreation must all rebuild"
);
}
}
#[test]
fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
let baseline_digest = DataUsageScanPlanDigest([3; 32]);
let old_cache = complete_set_usage_cache(&[("stable", 10)], baseline_digest);
let all_buckets = vec![bucket_info("stable"), bucket_info("new")];
let all_buckets = vec![bucket_info_with_created_time("stable"), bucket_info_with_created_time("new")];
assert!(
prepare_scoped_set_scan(
@@ -1302,7 +1631,7 @@ fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
#[test]
fn scoped_set_scan_requires_an_exact_complete_baseline() {
let baseline_digest = DataUsageScanPlanDigest([5; 32]);
let all_buckets = vec![bucket_info("dirty")];
let all_buckets = vec![bucket_info_with_created_time("dirty")];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
baseline_scan_plan_digest: Some(baseline_digest),
@@ -1323,6 +1652,10 @@ fn scoped_set_scan_requires_an_exact_complete_baseline() {
not_durable.info.last_update = None;
assert!(prepare_scoped_set_scan(&not_durable, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut unscoped_usage = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
unscoped_usage.cache.get_mut(DATA_USAGE_ROOT).expect("set root").objects = 1;
assert!(prepare_scoped_set_scan(&unscoped_usage, &all_buckets, &all_buckets, &scope, generation).is_none());
let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32]));
assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none());
@@ -1333,6 +1666,13 @@ fn scoped_set_scan_requires_an_exact_complete_baseline() {
};
let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none());
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &scope, generation).is_some());
let unidentified_buckets = vec![bucket_info("dirty")];
assert!(
prepare_scoped_set_scan(&complete, &unidentified_buckets, &unidentified_buckets, &scope, generation).is_some(),
"fully selected buckets are rebuilt without reusing an unproven incarnation"
);
let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
future_cache.info.next_cycle = generation.want_cycle.saturating_add(1);
@@ -1020,6 +1020,7 @@ mod serial_tests {
}
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
let expired_recovery_time = i128::from(i64::MAX / 2);
for case in [
CleanupCase::Persisted,
@@ -1117,7 +1118,7 @@ mod serial_tests {
.await
.expect("active unknown ownership must remain fenced after the transaction store was offline");
assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 1, 0));
let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX)
let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time)
.await
.expect("expired unknown ownership may use the provider's missing proof");
assert_eq!(
@@ -1166,7 +1167,7 @@ mod serial_tests {
assert_eq!(retained.recovered, 0);
assert_eq!(retained.retained + retained.failed, 1);
backend.set_remove_failure(false);
let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX)
let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time)
.await
.expect("expired recovery should delete the candidate after the backend becomes available");
assert_eq!(