mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 20:46:11 +00:00
chore(test): merge main before startup CAS coverage
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn emit(name: &str, value: &str) {
|
||||
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
|
||||
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
|
||||
let root = Path::new(&manifest).join("../..");
|
||||
// Cover dependency/common sources as well as this crate. HEAD/ref/index
|
||||
// changes must refresh identity even when no Rust source mtime changes.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={}", root.join(path).display());
|
||||
}
|
||||
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
|
||||
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
|
||||
git_paths.push(reference);
|
||||
}
|
||||
for path in git_paths {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
|
||||
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let mut features = std::env::vars()
|
||||
.filter_map(|(key, _)| {
|
||||
key.strip_prefix("CARGO_FEATURE_")
|
||||
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
features.sort();
|
||||
emit("COMMIT", &revision);
|
||||
emit("DIRTY", if dirty { "true" } else { "false" });
|
||||
emit("LOCK", &lock);
|
||||
emit("FEATURES", &features.join(","));
|
||||
for name in ["TARGET", "PROFILE"] {
|
||||
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
|
||||
}
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
|
||||
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
emit("RUSTFLAGS_HEX", &flags);
|
||||
}
|
||||
@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
@@ -88,13 +90,15 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
pub(crate) fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
@@ -413,6 +421,8 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
|
||||
@@ -35,11 +35,15 @@ where
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..45 {
|
||||
for _ in 0..90 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
if usage.is_complete_bucket_usage_snapshot()
|
||||
&& usage.usage_snapshot_converged != Some(false)
|
||||
&& usage.buckets_usage.contains_key(bucket)
|
||||
&& predicate(&usage)
|
||||
{
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
|
||||
@@ -16,15 +16,18 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -34,6 +37,76 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join("background-target-restart.json").exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some((directory, run)))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
@@ -195,8 +268,9 @@ mod tests {
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
let mut node_listings = Vec::with_capacity(clients.len());
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
@@ -243,8 +317,10 @@ mod tests {
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
listed_keys.sort();
|
||||
node_listings.push(listed_keys);
|
||||
}
|
||||
Ok(())
|
||||
Ok(node_listings)
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
@@ -808,6 +884,13 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
@@ -855,7 +938,7 @@ mod tests {
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start().await?;
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
@@ -996,7 +1079,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
cluster.start_node(1).await?;
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1274,7 +1357,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node(interruption_node).await?;
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
@@ -1373,7 +1456,7 @@ mod tests {
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
@@ -1381,11 +1464,31 @@ mod tests {
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": expected.shard_census,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": outage_key, "version_id": null,
|
||||
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
@@ -1432,6 +1535,31 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some((directory, run)) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
|
||||
"run_id": run.run_id, "source_revision": run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.join("background-target-restart.json"))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -369,26 +369,6 @@ struct SsecPassthroughRecord {
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
/// What a target write does when the bucket's persisted target set exists but
|
||||
/// cannot be decoded.
|
||||
///
|
||||
/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a
|
||||
/// configuration that could not be fully read, because re-serializing a
|
||||
/// partial in-memory view is the one mechanism by which a configured target
|
||||
/// really disappears. That rule guards against an *unintentional* overwrite,
|
||||
/// so an operator who names the hazard keeps a repair path
|
||||
/// (rustfs/backlog#2309); everything that does not name it stays refused.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum UnreadableTargetsPolicy {
|
||||
/// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`].
|
||||
#[default]
|
||||
FailClosed,
|
||||
/// Discard the unreadable set; the target being written becomes the whole
|
||||
/// configuration. Reachable only from an admin request that asked for it
|
||||
/// explicitly, and audited by the caller.
|
||||
Replace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
@@ -811,41 +791,23 @@ impl BucketTargetSys {
|
||||
bucket: &str,
|
||||
target: &BucketTarget,
|
||||
update: bool,
|
||||
unreadable_policy: UnreadableTargetsPolicy,
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
self.validate_target(bucket, target).await?;
|
||||
|
||||
let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?;
|
||||
let mut bucket_targets = self.targets_base_for_write(bucket).await?;
|
||||
|
||||
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
|
||||
|
||||
Ok(bucket_targets)
|
||||
}
|
||||
|
||||
/// The persisted target set a write merges into.
|
||||
///
|
||||
/// An absent configuration starts from the empty set. An unreadable one is
|
||||
/// refused, because re-serializing a partial view of a set this node could
|
||||
/// not decode is how a configured target disappears for good — unless the
|
||||
/// caller carries the operator's explicit
|
||||
/// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it
|
||||
/// deliberately (rustfs/backlog#2309).
|
||||
async fn targets_base_for_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
unreadable_policy: UnreadableTargetsPolicy,
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
/// Ordinary writes must not turn an unreadable cached snapshot into an
|
||||
/// empty configuration. Explicit repair belongs to the metadata transaction
|
||||
/// that can inspect the current persisted state.
|
||||
async fn targets_base_for_write(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Ok(targets),
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
|
||||
// The opt-in discards only a set this node genuinely cannot read.
|
||||
// A readable set still merges through the arm above, so the policy
|
||||
// can never drop a target that was visible here.
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
if unreadable_policy == UnreadableTargetsPolicy::Replace =>
|
||||
{
|
||||
Ok(BucketTargets::default())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -908,7 +870,9 @@ impl BucketTargetSys {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn upsert_target_entry(
|
||||
/// Merge a validated target into a caller-owned snapshot. The caller must
|
||||
/// protect that snapshot through persistence.
|
||||
pub fn upsert_target_entry(
|
||||
bucket_targets: &mut Vec<BucketTarget>,
|
||||
target: &BucketTarget,
|
||||
update: bool,
|
||||
@@ -1272,25 +1236,27 @@ impl BucketTargetSys {
|
||||
return (String::new(), false);
|
||||
};
|
||||
|
||||
{
|
||||
let targets_map = self.targets_map.read().await;
|
||||
if let Some(targets) = targets_map.get(bucket) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
}
|
||||
}
|
||||
let targets_map = self.targets_map.read().await;
|
||||
let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default();
|
||||
Self::remote_arn_for_targets(targets, target, depl_id)
|
||||
}
|
||||
|
||||
/// Resolve create idempotency against the snapshot the caller will persist.
|
||||
pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) {
|
||||
for tgt in targets {
|
||||
if tgt.target_type == target.target_type
|
||||
&& tgt.target_bucket == target.target_bucket
|
||||
&& target.endpoint == tgt.endpoint
|
||||
&& tgt
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|c| {
|
||||
let default_creds = Credentials::default();
|
||||
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return (tgt.arn.clone(), true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4371,47 +4337,19 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable
|
||||
/// `bucket-targets.json` left the bucket with no API repair path at all.
|
||||
/// The refusal is the default and stays the default; the operator's
|
||||
/// explicit opt-in is the only thing that discards the set, and it starts
|
||||
/// the replacement from empty rather than from a partial view of bytes
|
||||
/// this node never decoded.
|
||||
#[tokio::test]
|
||||
async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() {
|
||||
async fn an_unreadable_target_set_refuses_cached_writes() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-opt-in";
|
||||
sys.mark_targets_unreadable(bucket).await;
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
),
|
||||
"without the opt-in an unreadable target set must still refuse the write"
|
||||
);
|
||||
assert_eq!(
|
||||
UnreadableTargetsPolicy::default(),
|
||||
UnreadableTargetsPolicy::FailClosed,
|
||||
"a caller that says nothing must get the refusal"
|
||||
);
|
||||
|
||||
let base = sys
|
||||
.targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace)
|
||||
.await
|
||||
.expect("the explicit opt-in must let an operator replace an unreadable set");
|
||||
assert!(
|
||||
base.is_empty(),
|
||||
"the replacement must start from an empty set, never from a partial decode"
|
||||
);
|
||||
assert!(matches!(
|
||||
sys.targets_base_for_write(bucket).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// The opt-in is not a wipe switch. On a set this node can read, both
|
||||
/// policies take the same merge path, so a stray `replace-unreadable=true`
|
||||
/// cannot drop a visible target — which is what makes the flag safe to
|
||||
/// repeat in an operator's repair script.
|
||||
#[tokio::test]
|
||||
async fn the_opt_in_never_discards_a_readable_target_set() {
|
||||
async fn a_readable_target_set_remains_the_write_base() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-readable";
|
||||
let existing = repair_target(bucket, "keep");
|
||||
@@ -4419,14 +4357,8 @@ mod tests {
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), vec![existing.clone()]);
|
||||
|
||||
for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] {
|
||||
let base = sys
|
||||
.targets_base_for_write(bucket, policy)
|
||||
.await
|
||||
.expect("a readable target set must be readable under either policy");
|
||||
assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target");
|
||||
assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target");
|
||||
}
|
||||
let base = sys.targets_base_for_write(bucket).await.expect("read targets");
|
||||
assert_eq!(base.targets.len(), 1);
|
||||
assert_eq!(base.targets[0].arn, existing.arn);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
|
||||
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
|
||||
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
|
||||
@@ -747,7 +747,7 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
OperatorRequired(IlmRecoveryErrorCode),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryClaimBarrierState {
|
||||
transaction_id: Uuid,
|
||||
@@ -755,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionRecoveryClaimBarrier {
|
||||
state: Arc<TransitionRecoveryClaimBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl TransitionRecoveryClaimBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryClaimBarrierState {
|
||||
@@ -796,7 +796,7 @@ impl TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionRecoveryClaimBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -810,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -825,7 +825,7 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryTerminalBarrierState {
|
||||
transaction_id: Uuid,
|
||||
@@ -833,17 +833,17 @@ struct TransitionRecoveryTerminalBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionRecoveryTerminalBarrier {
|
||||
state: Arc<TransitionRecoveryTerminalBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl TransitionRecoveryTerminalBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryTerminalBarrierState {
|
||||
@@ -870,7 +870,7 @@ impl TransitionRecoveryTerminalBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionRecoveryTerminalBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -884,7 +884,7 @@ impl Drop for TransitionRecoveryTerminalBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -1242,7 +1242,7 @@ async fn process_transition_transaction_record_at(
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_before_transition_recovery_claim(current.transaction_id).await;
|
||||
match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await {
|
||||
Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
|
||||
@@ -1300,7 +1300,7 @@ async fn process_transition_transaction_record_at(
|
||||
};
|
||||
persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
|
||||
if let Some(source) = source_to_delete {
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_after_transition_recovery_terminal(source.transaction_id).await;
|
||||
delete_transition_transaction_record(api, &source).await?;
|
||||
}
|
||||
@@ -1322,7 +1322,7 @@ fn transition_recovery_control_identity(transaction: &TransitionTransaction, rec
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
|
||||
let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
|
||||
transition_recovery_control_identity(transaction, &record_name)
|
||||
@@ -1753,7 +1753,7 @@ pub async fn recover_transition_transaction_records(
|
||||
recover_transition_transaction_records_with_now(api, limit, marker, None).await
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn recover_transition_transaction_records_at(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
|
||||
@@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
|
||||
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
|
||||
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
|
||||
|
||||
@@ -1355,6 +1355,7 @@ impl LocalDiskWrapper {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path_for_io(volume, path)
|
||||
}
|
||||
|
||||
@@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
|
||||
@@ -735,7 +735,9 @@ const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_fa
|
||||
/// to replace. Best effort — the rename that follows fails closed — but a
|
||||
/// recurring signal means heal is stuck on that drive.
|
||||
const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed";
|
||||
#[cfg(unix)]
|
||||
const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total";
|
||||
#[cfg(unix)]
|
||||
const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total";
|
||||
// io_uring read-backend gray-release observability (rustfs/backlog#1172).
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -932,10 +934,15 @@ const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_
|
||||
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
|
||||
)]
|
||||
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
|
||||
#[cfg(any(unix, test))]
|
||||
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
|
||||
#[cfg(any(unix, test))]
|
||||
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
|
||||
#[cfg(any(unix, test))]
|
||||
const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD";
|
||||
#[cfg(any(unix, test))]
|
||||
const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy";
|
||||
#[cfg(any(unix, test))]
|
||||
const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy";
|
||||
|
||||
/// Legacy binary switch for commit-point durability (fsync writes and renames).
|
||||
@@ -951,6 +958,7 @@ const DEFAULT_RUSTFS_DRIVE_SYNC_ENABLE: bool = true;
|
||||
/// See docs/operations/durability-modes.md for the power-loss guarantee matrix.
|
||||
const ENV_RUSTFS_DURABILITY_MODE: &str = "RUSTFS_DURABILITY_MODE";
|
||||
|
||||
#[cfg(any(unix, test))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum LocalReadCopyMethod {
|
||||
MmapCopy,
|
||||
@@ -1359,15 +1367,18 @@ cached_read_env! {
|
||||
|
||||
cached_read_env! {
|
||||
/// Whether mmap reads should fault the mapping in with `MAP_POPULATE`.
|
||||
#[cfg(any(unix, test))]
|
||||
fn mmap_populate_enabled() -> bool =
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE);
|
||||
}
|
||||
|
||||
#[cfg(any(unix, test))]
|
||||
fn should_populate_mmap_read(length: usize) -> bool {
|
||||
length > 0 && mmap_populate_enabled()
|
||||
}
|
||||
|
||||
cached_read_env! {
|
||||
#[cfg(any(unix, test))]
|
||||
fn local_read_copy_method() -> LocalReadCopyMethod = {
|
||||
let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY);
|
||||
match method.as_str() {
|
||||
@@ -2010,7 +2021,7 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, unix))]
|
||||
fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
INLINE_BEFORE_FILE_SYNC_ADMISSION
|
||||
.lock()
|
||||
@@ -2257,7 +2268,7 @@ fn should_remove_staged_meta_before_commit(_dst_path: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(not(test), not(windows)))]
|
||||
fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
|
||||
false
|
||||
}
|
||||
@@ -3758,6 +3769,7 @@ struct FdKey {
|
||||
/// The generation fence and explicit mutation invalidation keep the snapshot
|
||||
/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
|
||||
/// metadata syscall without weakening replacement/heal semantics.
|
||||
#[cfg(unix)]
|
||||
struct FdCacheEntry {
|
||||
/// An independently cloneable descriptor for the immutable shard inode.
|
||||
file: Arc<std::fs::File>,
|
||||
@@ -5508,6 +5520,7 @@ impl LocalDisk {
|
||||
local_disk_bucket_path(&self.root, bucket)
|
||||
}
|
||||
|
||||
#[cfg(any(unix, test))]
|
||||
pub(crate) fn get_object_path_for_io(&self, bucket: &str, key: &str) -> Result<PathBuf> {
|
||||
self.io_get_object_path(bucket, key)
|
||||
}
|
||||
@@ -19351,11 +19364,17 @@ mod test {
|
||||
path_resolve_stage: "path",
|
||||
metadata_lookup_stage: "metadata_lookup",
|
||||
metadata_validate_stage: "metadata_validate",
|
||||
#[cfg(unix)]
|
||||
blocking_wait_stage: "blocking_wait",
|
||||
#[cfg(unix)]
|
||||
blocking_task_stage: "blocking_task",
|
||||
#[cfg(unix)]
|
||||
file_open_stage: "file_open",
|
||||
#[cfg(unix)]
|
||||
mmap_map_stage: "mmap_map",
|
||||
#[cfg(unix)]
|
||||
mmap_copy_stage: "mmap_copy",
|
||||
#[cfg(unix)]
|
||||
direct_read_copy_stage: "direct_read_copy",
|
||||
};
|
||||
|
||||
|
||||
@@ -17,13 +17,14 @@
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
use super::should_fail_local_inline_rollback_hardlink;
|
||||
use super::{
|
||||
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
|
||||
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
|
||||
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
|
||||
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
|
||||
skip_access_checks,
|
||||
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
|
||||
@@ -88,6 +89,7 @@ fn rollback_inline_metadata_commit_std(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
|
||||
@@ -198,11 +198,17 @@ pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path_resolve_stage: &'static str,
|
||||
pub(crate) metadata_lookup_stage: &'static str,
|
||||
pub(crate) metadata_validate_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_wait_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_task_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) file_open_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_map_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_copy_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) direct_read_copy_stage: &'static str,
|
||||
}
|
||||
|
||||
@@ -949,6 +955,7 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io_if_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
@@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
|
||||
#[cfg(unix)]
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
@@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
|
||||
@@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
.lock()
|
||||
@@ -237,6 +240,7 @@ pub(crate) mod fsync_dir_recorder {
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
|
||||
}
|
||||
|
||||
@@ -13,12 +13,15 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
@@ -406,11 +409,17 @@ async fn open_disk_reader(
|
||||
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
|
||||
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
|
||||
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
#[cfg(unix)]
|
||||
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
#[cfg(unix)]
|
||||
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
|
||||
#[cfg(unix)]
|
||||
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
#[cfg(unix)]
|
||||
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
|
||||
#[cfg(unix)]
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
#[cfg(unix)]
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
let mmap_result = {
|
||||
|
||||
@@ -56,6 +56,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
#[cfg(feature = "test-util")]
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::format::FormatV3;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the barrier for the next rejected transition cleanup.
|
||||
pub fn install() -> Self {
|
||||
@@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
|
||||
/// Default polling cadence used by the `wait_for_*` helpers.
|
||||
#[cfg(feature = "test-util")]
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// A fault to inject into [`MockWarmBackend`] operations.
|
||||
@@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockPutBarrier {
|
||||
/// Wait until the remote body is stored and the PUT is paused before returning.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -226,6 +237,7 @@ impl MockPutBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockPutBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -258,10 +270,12 @@ impl Drop for MockGetBarrier {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses and then fails a mock tier DELETE.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockRemoveBarrier {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockRemoveBarrier {
|
||||
/// Wait until DELETE reaches the deterministic failure point.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -283,6 +297,7 @@ impl MockRemoveBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockRemoveBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -306,6 +321,7 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause after the next tier PUT stores its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
|
||||
let state = Arc::new(MockPutBarrierState::default());
|
||||
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
|
||||
@@ -313,6 +329,7 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Pause and then fail the next DELETE after it reaches the backend.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
|
||||
let state = Arc::new(MockRemoveBarrierState::default());
|
||||
let mut barrier = self.inner.remove_barrier.lock().await;
|
||||
@@ -323,6 +340,7 @@ impl MockWarmBackend {
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then return an error
|
||||
/// after the test releases it.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
|
||||
let state = Arc::new(MockGetBarrierState {
|
||||
fail_after_release: true,
|
||||
@@ -343,6 +361,7 @@ impl MockWarmBackend {
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_faults(&self, faults: FaultConfig) {
|
||||
*self.inner.faults.lock().await = faults;
|
||||
}
|
||||
@@ -353,6 +372,7 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Toggle "HTTP 5xx" server errors on every operation.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_server_error(&self, server_error: bool) {
|
||||
self.inner.faults.lock().await.server_error = server_error;
|
||||
}
|
||||
@@ -363,11 +383,13 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Set (or clear, with `None`) injected latency applied before each op.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_latency(&self, latency: Option<Duration>) {
|
||||
self.inner.faults.lock().await.latency = latency;
|
||||
}
|
||||
|
||||
/// Clear all injected faults, restoring healthy behaviour.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_faults(&self) {
|
||||
*self.inner.faults.lock().await = FaultConfig::default();
|
||||
}
|
||||
@@ -375,6 +397,7 @@ impl MockWarmBackend {
|
||||
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
|
||||
/// the complete body. This models a backend that incorrectly accepts a
|
||||
/// truncated stream while still returning success.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
|
||||
*self.inner.put_read_limit.lock().await = limit;
|
||||
}
|
||||
@@ -395,12 +418,14 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Reject the next non-empty remote version validation without changing
|
||||
/// subsequent exact-version backend cleanup behavior.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn reject_next_non_empty_remote_version_validation(&self) {
|
||||
self.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
@@ -438,6 +463,7 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Clear the operation log without touching stored objects or faults.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_op_log(&self) {
|
||||
self.inner.op_log.lock().await.clear();
|
||||
}
|
||||
@@ -459,11 +485,13 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs passed to successful tier removes.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn remove_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
@@ -475,6 +503,7 @@ impl MockWarmBackend {
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn get_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -486,6 +515,7 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Number of `put` calls recorded.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -499,6 +529,7 @@ impl MockWarmBackend {
|
||||
// ---- storage inspection ---------------------------------------------
|
||||
|
||||
/// Whether the backend currently stores `object`.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn contains(&self, object: &str) -> bool {
|
||||
self.inner.objects.lock().await.contains_key(object)
|
||||
}
|
||||
@@ -509,11 +540,13 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// A clone of the stored object, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
|
||||
self.inner.objects.lock().await.get(object).cloned()
|
||||
}
|
||||
|
||||
/// A clone of the raw bytes stored for `object`, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
|
||||
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
|
||||
}
|
||||
@@ -538,6 +571,7 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until `object` is absent from the backend, or `timeout` elapses.
|
||||
/// Returns `true` if the object disappeared within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -553,6 +587,7 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until the backend holds exactly `expected` objects, or `timeout`
|
||||
/// elapses. Returns `true` if the count was reached within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
|
||||
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
|
||||
/// free-version count.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionMeta {
|
||||
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
|
||||
pub status: String,
|
||||
@@ -860,6 +896,7 @@ pub struct TransitionMeta {
|
||||
pub free_version_count: usize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
|
||||
// with the position recorded in the disk's own format.json, so derive the
|
||||
@@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
/// The free-version metadata removal lands asynchronously after the remote
|
||||
/// object disappears, so callers typically poll via
|
||||
/// [`wait_for_free_version_absence`] instead of asserting a single read.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
|
||||
let Some(disk) = open_disk(disk_path).await else {
|
||||
return 0;
|
||||
@@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
|
||||
/// fields are taken from the newest version that carries a transition record;
|
||||
/// if no version is transitioned, they are taken from the current version (and
|
||||
/// will be empty).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
|
||||
let disk = open_disk(disk_path).await?;
|
||||
let data = disk
|
||||
@@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
/// disk is missing the object or disagrees — this is the shard-consistency
|
||||
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
|
||||
/// four-tuple plus free-version count must match across all erasure shards).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
|
||||
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
|
||||
|
||||
@@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
|
||||
|
||||
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
|
||||
/// elapses. Returns `true` if the free versions drained within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -1040,6 +1081,44 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
|
||||
let backend = MockWarmBackend::new();
|
||||
let metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("project".to_string(), "archive".to_string()),
|
||||
]);
|
||||
let version = backend
|
||||
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
|
||||
.await
|
||||
.expect("mock PUT should preserve remote metadata");
|
||||
assert_eq!(backend.metadata("object").await, Some(metadata));
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe stored object"),
|
||||
TransitionCandidateProbe::VersionedPresent(version)
|
||||
);
|
||||
|
||||
backend.external_remove("object").await;
|
||||
assert_eq!(backend.metadata("object").await, None);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe removed object"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
let operations = backend.op_log().await;
|
||||
assert!(
|
||||
operations
|
||||
.iter()
|
||||
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
|
||||
);
|
||||
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_preserves_fault_fail_closed_behavior() {
|
||||
let backend = MockWarmBackend::new();
|
||||
|
||||
@@ -111,10 +111,11 @@ use crate::disk::{
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
use crate::io_support::bitrot::{
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
|
||||
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
|
||||
object_mmap_read_max_length,
|
||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
|
||||
create_deferred_bitrot_reader_with_stripe_handle,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
|
||||
use crate::set_disk::runtime_sources;
|
||||
use crate::set_disk::shard_source::ShardReadCost;
|
||||
use crate::storage_api_contracts::object::ObjectOperations;
|
||||
|
||||
@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
|
||||
|
||||
@@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
}
|
||||
|
||||
async fn transition_cleanup_store(ctx: &Arc<crate::runtime::instance::InstanceContext>) -> Option<Arc<ECStore>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
pause_transition_cleanup_store().await;
|
||||
|
||||
transition_object_store(ctx).await
|
||||
@@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
#[derive(Default)]
|
||||
struct TransitionCleanupStoreBarrierState {
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
/// One-shot test barrier placed before transition cleanup resolves its ECStore.
|
||||
pub(crate) struct TransitionCleanupStoreBarrier {
|
||||
state: Arc<TransitionCleanupStoreBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCleanupStoreBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the process-local barrier for the next cleanup-store resolution.
|
||||
pub(crate) fn install() -> Self {
|
||||
@@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for TransitionCleanupStoreBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn pause_transition_cleanup_store() {
|
||||
let barrier = TRANSITION_CLEANUP_STORE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
struct TransitionUploadedCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
@@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionUploadedCommitBarrier {
|
||||
state: Arc<TransitionUploadedCommitBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionUploadedCommitBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl TransitionUploadedCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedCommitBarrierState {
|
||||
@@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionUploadedCommitBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) {
|
||||
let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
upload_cleanup.update_cleanup_transaction(&transaction);
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_after_transition_uploaded_persisted(bucket, object).await;
|
||||
|
||||
let commit_opts = opts.as_commit_opts();
|
||||
@@ -12672,6 +12672,65 @@ mod metadata_mutation_generation_tests {
|
||||
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
||||
async fn segment_observation_equal_size_mutations_retire_metadata_generation() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "segment-observation-bucket";
|
||||
let object = "hot/object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create segment fixture bucket");
|
||||
}
|
||||
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await;
|
||||
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
|
||||
let mut replacement = PutObjReader::from_vec(b"after!".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("commit same-length replacement with normal owner locking");
|
||||
assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation");
|
||||
assert_retired(&set_disks, &old_key).await;
|
||||
drop(probe);
|
||||
let after = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read replacement metadata");
|
||||
assert_eq!(before.size, after.size);
|
||||
assert_ne!(before.etag, after.etag, "equal size is not equal content");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read replacement body through the owner");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("drain replacement body");
|
||||
assert_eq!(body, b"after!");
|
||||
drop(reader);
|
||||
|
||||
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await;
|
||||
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit metadata-only mutation with normal owner locking");
|
||||
assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences");
|
||||
assert_retired(&set_disks, &old_key).await;
|
||||
let after = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read committed metadata-only mutation");
|
||||
assert_eq!(before.size, after.size);
|
||||
assert_eq!(before.etag, after.etag);
|
||||
assert!(!before.user_defined.contains_key("x-amz-meta-segment"));
|
||||
assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(metadata_cache_invalidation_probe)]
|
||||
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
|
||||
|
||||
@@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation(
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
|
||||
data_movement_multipart_discovery_counts()
|
||||
.lock()
|
||||
@@ -246,7 +246,7 @@ impl ECStore {
|
||||
.insert(self.id, 0);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
|
||||
data_movement_multipart_discovery_counts()
|
||||
.lock()
|
||||
|
||||
@@ -54,6 +54,15 @@ pub enum Error {
|
||||
#[error("Heal task execution failed: {message}")]
|
||||
TaskExecutionFailed { message: String },
|
||||
|
||||
/// The current page already exhausted its local retry budget. Retrying
|
||||
/// the enclosing bucket would replay pages whose results were counted.
|
||||
#[error("Heal listing failed for bucket {bucket}: {source}")]
|
||||
HealListingFailed {
|
||||
bucket: String,
|
||||
#[source]
|
||||
source: Box<Error>,
|
||||
},
|
||||
|
||||
#[error("Invalid heal type: {heal_type}")]
|
||||
InvalidHealType { heal_type: String },
|
||||
|
||||
|
||||
@@ -447,6 +447,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
None,
|
||||
@@ -463,6 +464,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"running".to_string(),
|
||||
Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
|
||||
@@ -479,6 +481,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"finished".to_string(),
|
||||
None,
|
||||
@@ -495,6 +498,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task cancelled".to_string()),
|
||||
@@ -511,6 +515,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some("heal task timed out".to_string()),
|
||||
@@ -527,6 +532,7 @@ impl HealChannelProcessor {
|
||||
progress,
|
||||
next_seq,
|
||||
min_seq,
|
||||
..
|
||||
}) => (
|
||||
"stopped".to_string(),
|
||||
Some(error),
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::heal::{
|
||||
outcome::HealTaskOutcome,
|
||||
progress::{HealProgress, HealStatistics},
|
||||
resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
|
||||
storage::HealStorageAPI,
|
||||
@@ -185,6 +186,7 @@ fn record_displaced_terminal(
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
outcome: None,
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
@@ -268,6 +270,7 @@ async fn publish_completed_heal(
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealTaskReport {
|
||||
pub outcome: Option<Arc<HealTaskOutcome>>,
|
||||
pub status: HealTaskStatus,
|
||||
pub result_items: Vec<HealResultItem>,
|
||||
pub result_items_truncated: bool,
|
||||
@@ -285,6 +288,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
|
||||
let window = task.get_result_items_since(since).await;
|
||||
HealTaskReport {
|
||||
status: task.get_status().await,
|
||||
outcome: Some(Arc::new(task.get_outcome().await)),
|
||||
result_items: window.items,
|
||||
// The legacy flag stays set once anything was evicted; a lagging
|
||||
// incremental cursor additionally marks this response truncated so
|
||||
@@ -298,6 +302,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
|
||||
|
||||
fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
|
||||
HealTaskReport {
|
||||
outcome: None,
|
||||
status,
|
||||
result_items: Vec::new(),
|
||||
result_items_truncated: false,
|
||||
@@ -325,6 +330,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
};
|
||||
HealTaskReport {
|
||||
status: completed.status.clone(),
|
||||
outcome: completed.outcome.clone(),
|
||||
result_items,
|
||||
result_items_truncated: completed.result_items_truncated || lagged,
|
||||
progress: completed.progress.clone(),
|
||||
|
||||
@@ -83,6 +83,7 @@ pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) progress: Option<HealProgress>,
|
||||
pub(super) outcome: Option<Arc<HealTaskOutcome>>,
|
||||
pub(super) retained_bytes: std::sync::OnceLock<usize>,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
@@ -105,6 +106,7 @@ impl CompletedHealStatus {
|
||||
fn measure_retained_bytes(&self) -> usize {
|
||||
let mut bytes = size_of::<Self>();
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => {}
|
||||
HealType::Bucket { bucket } => add(bucket.capacity()),
|
||||
@@ -209,6 +211,7 @@ impl CompletedHealStatus {
|
||||
heal_type: task.heal_type.clone(),
|
||||
status,
|
||||
progress: Some(task.get_progress().await),
|
||||
outcome: Some(Arc::new(task.get_outcome().await)),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
|
||||
@@ -298,6 +298,7 @@ impl HealManager {
|
||||
if cancelled_completion {
|
||||
completed_status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
|
||||
@@ -103,6 +103,7 @@ struct MockStorage;
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
outcome: None,
|
||||
heal_type: HealType::Cluster,
|
||||
status: HealTaskStatus::Completed,
|
||||
progress: Some(HealProgress {
|
||||
@@ -287,6 +288,59 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
|
||||
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
|
||||
use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
|
||||
let bucket = "canonical-outcome-cancel-before-finish";
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
let retention_hook = Arc::new(CompletedRetentionHook::default());
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.to_string(), retention_hook.clone());
|
||||
hooks.insert(task_id.clone(), retention_hook.clone());
|
||||
}
|
||||
let finish_hook = Arc::new(OutcomeFinishTestHook {
|
||||
task_id: task_id.clone(),
|
||||
reached: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
*OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone());
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified())
|
||||
.await
|
||||
.expect("storage started");
|
||||
retention_hook.execute.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified())
|
||||
.await
|
||||
.expect("storage returned before outcome finalization");
|
||||
manager.cancel_task(&alias).await.expect("cancel wins publication");
|
||||
finish_hook.release.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler completes cancelled handoff");
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(
|
||||
report.outcome.as_ref().expect("frozen outcome").execution,
|
||||
HealExecutionOutcome::Aborted(HealAbortReason::Cancelled)
|
||||
);
|
||||
}
|
||||
retention_hook.finish.notify_one();
|
||||
*OUTCOME_FINISH_TEST_HOOK.lock().await = None;
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != bucket && key != &task_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
let bucket = "completed-retention-retry-cancel";
|
||||
@@ -325,6 +379,10 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(
|
||||
report.outcome.as_ref().expect("cancelled outcome retained").execution,
|
||||
crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled)
|
||||
);
|
||||
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
|
||||
}
|
||||
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
|
||||
@@ -391,6 +449,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
.expect("scheduler archives terminal");
|
||||
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
|
||||
let expected = task.get_progress().await;
|
||||
let expected_outcome = task.get_outcome().await;
|
||||
for token in [&task_id, &alias] {
|
||||
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
|
||||
let report = manager
|
||||
@@ -398,6 +457,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
.await
|
||||
.expect("terminal token remains queryable at handoff");
|
||||
assert_eq!(report.progress.as_ref(), Some(&expected));
|
||||
assert_eq!(report.outcome.as_deref(), Some(&expected_outcome));
|
||||
assert!(report.result_items.is_empty());
|
||||
match outcome {
|
||||
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
|
||||
@@ -1976,6 +2036,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
task_id,
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type,
|
||||
status: HealTaskStatus::Retrying {
|
||||
@@ -2700,6 +2761,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
task_id.clone(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Retrying {
|
||||
@@ -2737,6 +2799,7 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -2768,6 +2831,7 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod outcome;
|
||||
pub mod progress;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
|
||||
@@ -33,6 +33,9 @@ use std::collections::HashMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Explicit pending migration; never activates the production writer or GC.
|
||||
pub mod migration;
|
||||
|
||||
// Root-level control files avoid requiring a new directory before the first
|
||||
// atomic commit. They remain inside the storage owner's metadata volume.
|
||||
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
|
||||
@@ -66,6 +69,23 @@ struct Manifest {
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
|
||||
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
|
||||
bytes.extend_from_slice(MAGIC);
|
||||
bytes.push(VERSION);
|
||||
bytes.extend_from_slice(owner.as_bytes());
|
||||
bytes.extend_from_slice(&sequence.to_le_bytes());
|
||||
bytes.extend_from_slice(
|
||||
&u64::try_from(payload.len())
|
||||
.map_err(|_| SnapshotError::TooLarge)?
|
||||
.to_le_bytes(),
|
||||
);
|
||||
bytes.extend_from_slice(&Sha256::digest(payload));
|
||||
bytes.extend_from_slice(&Sha256::digest(&bytes));
|
||||
Self::decode(&bytes, payload.len())?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
|
||||
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,305 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Execution results are separate from repair responsibility. A legacy
|
||||
//! successful storage call supplies no authoritative repair receipt.
|
||||
|
||||
use std::{collections::VecDeque, time::SystemTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
const MAX_OUTCOME_ITEMS: usize = 128;
|
||||
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
|
||||
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealObjectKind {
|
||||
Object,
|
||||
Metadata,
|
||||
Decode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectIdentity {
|
||||
pub kind: HealObjectKind,
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
/// The requested version; None remains unresolved, never an absence proof.
|
||||
pub version_id: Option<String>,
|
||||
pub bucket_incarnation_id: Option<Uuid>,
|
||||
pub pool_index: Option<usize>,
|
||||
pub set_index: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealDeferredReason {
|
||||
DanglingDeleteGrace,
|
||||
TransientUsageCache,
|
||||
TransientExistenceCheck,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealFailureClass {
|
||||
Recoverable,
|
||||
RetryExhausted,
|
||||
Permanent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HealObjectDisposition {
|
||||
/// The legacy storage response does not prove the requested check or commit.
|
||||
Unknown,
|
||||
Repaired,
|
||||
VerifiedHealthy,
|
||||
AuthoritativelyAbsent,
|
||||
Deferred {
|
||||
reason: HealDeferredReason,
|
||||
retry_not_before: Option<SystemTime>,
|
||||
},
|
||||
Failed(HealFailureClass),
|
||||
Cancelled,
|
||||
DryRunObserved,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectOutcome {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
impl HealObjectOutcome {
|
||||
fn retained_bytes(&self) -> usize {
|
||||
size_of::<Self>()
|
||||
.saturating_add(self.identity.bucket.capacity())
|
||||
.saturating_add(self.identity.object.capacity())
|
||||
.saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity))
|
||||
.saturating_add(self.detail.as_ref().map_or(0, String::capacity))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealTraversalCoverage {
|
||||
#[default]
|
||||
Unknown,
|
||||
Partial,
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HealAbortReason {
|
||||
Cancelled,
|
||||
Deadline,
|
||||
Untraversable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum HealExecutionOutcome {
|
||||
#[default]
|
||||
Pending,
|
||||
Running,
|
||||
Completed,
|
||||
CompletedWithErrors,
|
||||
Aborted(HealAbortReason),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealOutcomeCounters {
|
||||
pub processed: u64,
|
||||
pub healed: u64,
|
||||
pub unchanged: u64,
|
||||
/// Deferred, cancelled, dry-run and unverified results remain unresolved.
|
||||
pub skipped: u64,
|
||||
pub failed: u64,
|
||||
pub unknown: u64,
|
||||
pub attempt_failures: u64,
|
||||
pub overflowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct HealTaskOutcome {
|
||||
pub execution: HealExecutionOutcome,
|
||||
pub coverage: HealTraversalCoverage,
|
||||
pub counters: HealOutcomeCounters,
|
||||
/// A bounded diagnostic window, not a complete responsibility ledger.
|
||||
pub objects: VecDeque<HealObjectOutcome>,
|
||||
pub objects_truncated: bool,
|
||||
retained_object_bytes: usize,
|
||||
untraversable: bool,
|
||||
}
|
||||
|
||||
impl HealTaskOutcome {
|
||||
pub(crate) fn start(&mut self) {
|
||||
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
self.execution = HealExecutionOutcome::Running;
|
||||
}
|
||||
self.coverage = HealTraversalCoverage::Partial;
|
||||
}
|
||||
|
||||
pub(crate) fn attempt_failed(&mut self) {
|
||||
self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures);
|
||||
}
|
||||
|
||||
pub(crate) fn mark_untraversable(&mut self) {
|
||||
self.untraversable = true;
|
||||
self.coverage = HealTraversalCoverage::Partial;
|
||||
}
|
||||
|
||||
pub(crate) fn finish(&mut self, abort: Option<HealAbortReason>) {
|
||||
if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
|
||||
return;
|
||||
}
|
||||
let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable));
|
||||
self.execution = match abort {
|
||||
Some(reason) => HealExecutionOutcome::Aborted(reason),
|
||||
None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors,
|
||||
None => HealExecutionOutcome::Completed,
|
||||
};
|
||||
self.coverage = if abort.is_none() && !self.counters.overflowed {
|
||||
HealTraversalCoverage::Complete
|
||||
} else {
|
||||
HealTraversalCoverage::Partial
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) fn record(&mut self, mut item: HealObjectOutcome) {
|
||||
use super::progress::increment_counter;
|
||||
let counters = &mut self.counters;
|
||||
counters.overflowed |= !increment_counter(&mut counters.processed);
|
||||
let counter = match item.disposition {
|
||||
HealObjectDisposition::Repaired => &mut counters.healed,
|
||||
HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
|
||||
HealObjectDisposition::Failed(_) => &mut counters.failed,
|
||||
HealObjectDisposition::Unknown => {
|
||||
counters.overflowed |= !increment_counter(&mut counters.unknown);
|
||||
&mut counters.skipped
|
||||
}
|
||||
_ => &mut counters.skipped,
|
||||
};
|
||||
counters.overflowed |= !increment_counter(counter);
|
||||
if let Some(detail) = &mut item.detail {
|
||||
let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES);
|
||||
while !detail.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
self.objects_truncated |= end < detail.len();
|
||||
detail.truncate(end);
|
||||
detail.shrink_to_fit();
|
||||
}
|
||||
let bytes = item.retained_bytes();
|
||||
if bytes > MAX_OUTCOME_BYTES {
|
||||
self.objects_truncated = true;
|
||||
return;
|
||||
}
|
||||
while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES {
|
||||
let Some(oldest) = self.objects.pop_front() else { break };
|
||||
self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes());
|
||||
self.objects_truncated = true;
|
||||
}
|
||||
self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes);
|
||||
self.objects.push_back(item);
|
||||
}
|
||||
|
||||
pub(crate) fn retained_bytes(&self) -> usize {
|
||||
size_of::<Self>()
|
||||
.saturating_add(self.retained_object_bytes)
|
||||
.saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod canonical_outcome_tests {
|
||||
use super::*;
|
||||
|
||||
fn item(disposition: HealObjectDisposition) -> HealObjectOutcome {
|
||||
HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_categories_have_one_terminal_count() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for disposition in [
|
||||
HealObjectDisposition::Unknown,
|
||||
HealObjectDisposition::Repaired,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
HealObjectDisposition::AuthoritativelyAbsent,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
},
|
||||
HealObjectDisposition::Failed(HealFailureClass::Permanent),
|
||||
HealObjectDisposition::Cancelled,
|
||||
HealObjectDisposition::DryRunObserved,
|
||||
] {
|
||||
outcome.record(item(disposition));
|
||||
}
|
||||
let c = &outcome.counters;
|
||||
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
|
||||
assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
for _ in 0..MAX_OUTCOME_ITEMS {
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
}
|
||||
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
|
||||
assert!(!outcome.objects_truncated);
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
|
||||
assert!(outcome.objects_truncated);
|
||||
let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
|
||||
oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES);
|
||||
outcome.record(oversized);
|
||||
assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count"));
|
||||
assert_eq!(outcome.counters.failed, 1);
|
||||
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
|
||||
for _ in 0..MAX_OUTCOME_ITEMS {
|
||||
let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
|
||||
failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES));
|
||||
outcome.record(failed);
|
||||
}
|
||||
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
|
||||
assert!(outcome.objects.iter().all(|item| {
|
||||
item.detail
|
||||
.as_ref()
|
||||
.is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES)
|
||||
}));
|
||||
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.counters.processed = u64::MAX;
|
||||
outcome.record(item(HealObjectDisposition::Unknown));
|
||||
outcome.finish(None);
|
||||
assert!(outcome.counters.overflowed);
|
||||
assert_eq!(outcome.counters.processed, u64::MAX);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,10 @@
|
||||
use crate::heal::{
|
||||
DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _,
|
||||
erasure_healer::target_outcomes_complete,
|
||||
outcome::{
|
||||
HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind,
|
||||
HealObjectOutcome, HealTaskOutcome,
|
||||
},
|
||||
progress::HealProgress,
|
||||
resume::{
|
||||
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
|
||||
@@ -43,6 +47,26 @@ use uuid::Uuid;
|
||||
|
||||
use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct OutcomeFinishTestHook {
|
||||
pub(crate) task_id: String,
|
||||
pub(crate) reached: tokio::sync::Notify,
|
||||
pub(crate) release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_outcome_finish(task_id: &str) {
|
||||
let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
|
||||
if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_TASK: &str = "task";
|
||||
const LOG_SUBSYSTEM_OBJECT: &str = "object";
|
||||
@@ -394,6 +418,7 @@ pub struct HealTask {
|
||||
pub status: Arc<RwLock<HealTaskStatus>>,
|
||||
/// Progress tracking
|
||||
pub progress: Arc<RwLock<HealProgress>>,
|
||||
outcome: Arc<RwLock<HealTaskOutcome>>,
|
||||
/// Result items collected from storage heal calls, each stamped with a
|
||||
/// monotonically increasing sequence number for incremental consumption
|
||||
/// (the client passes the last seen seq back and receives only newer
|
||||
@@ -460,6 +485,7 @@ impl HealTask {
|
||||
result_items_truncated: Arc::new(AtomicBool::new(false)),
|
||||
batch_failure: Arc::new(RwLock::new(None)),
|
||||
batch_failure_recorded: Arc::new(AtomicBool::new(false)),
|
||||
outcome: Arc::new(RwLock::new(HealTaskOutcome::default())),
|
||||
created_at: request.created_at,
|
||||
enqueued_at: request.enqueued_at,
|
||||
started_at: Arc::new(RwLock::new(None)),
|
||||
@@ -507,6 +533,66 @@ impl HealTask {
|
||||
self.heal_type.kind_label()
|
||||
}
|
||||
|
||||
pub async fn get_outcome(&self) -> HealTaskOutcome {
|
||||
self.outcome.read().await.clone()
|
||||
}
|
||||
|
||||
fn outcome_identity(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
pool_index: Option<usize>,
|
||||
set_index: Option<usize>,
|
||||
) -> HealObjectIdentity {
|
||||
HealObjectIdentity {
|
||||
kind: match self.heal_type {
|
||||
HealType::Metadata { .. } => HealObjectKind::Metadata,
|
||||
HealType::ECDecode { .. } => HealObjectKind::Decode,
|
||||
_ => HealObjectKind::Object,
|
||||
},
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: None,
|
||||
pool_index,
|
||||
set_index,
|
||||
}
|
||||
}
|
||||
|
||||
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
||||
let (bucket, object, version) = match &self.heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
| HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => (bucket, object, version_id.as_deref()),
|
||||
HealType::Metadata { bucket, object } => (bucket, object, None),
|
||||
_ => return None,
|
||||
};
|
||||
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
|
||||
}
|
||||
|
||||
async fn record_deferred_object(&self, reason: HealDeferredReason) {
|
||||
if let Some(identity) = self.single_object_identity() {
|
||||
let mut outcome = self.outcome.write().await;
|
||||
outcome.attempt_failed();
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition: HealObjectDisposition::Deferred {
|
||||
reason,
|
||||
retry_not_before: None,
|
||||
},
|
||||
detail: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_batch_failure(&self) -> bool {
|
||||
self.batch_failure_recorded.load(Ordering::Acquire)
|
||||
}
|
||||
@@ -634,6 +720,7 @@ impl HealTask {
|
||||
}
|
||||
|
||||
async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
|
||||
self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await;
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -733,6 +820,8 @@ impl HealTask {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_deferred_object(HealDeferredReason::TransientUsageCache).await;
|
||||
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -755,6 +844,8 @@ impl HealTask {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await;
|
||||
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
@@ -801,6 +892,7 @@ impl HealTask {
|
||||
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
|
||||
#[hotpath::measure]
|
||||
pub async fn execute(&self) -> Result<()> {
|
||||
self.outcome.write().await.start();
|
||||
// update status and timestamps atomically to avoid race conditions
|
||||
let now = SystemTime::now();
|
||||
let start_instant = Instant::now();
|
||||
@@ -860,6 +952,45 @@ impl HealTask {
|
||||
HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pause_outcome_finish(&self.id).await;
|
||||
{
|
||||
let mut outcome = self.outcome.write().await;
|
||||
if outcome.counters.processed == 0
|
||||
&& let Some(identity) = self.single_object_identity()
|
||||
{
|
||||
let disposition = match &result {
|
||||
Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved,
|
||||
Ok(()) => HealObjectDisposition::Unknown,
|
||||
Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled,
|
||||
Err(Error::TaskTimeout) => HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
},
|
||||
Err(error) => {
|
||||
outcome.attempt_failed();
|
||||
HealObjectDisposition::Failed(if error.is_recoverable_heal() {
|
||||
HealFailureClass::Recoverable
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
})
|
||||
}
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: result.as_ref().err().map(ToString::to_string),
|
||||
});
|
||||
}
|
||||
let abort = match &result {
|
||||
Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled),
|
||||
Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline),
|
||||
Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable),
|
||||
_ => None,
|
||||
};
|
||||
outcome.finish(abort);
|
||||
}
|
||||
|
||||
// update completed time and status
|
||||
{
|
||||
let mut completed_at = self.completed_at.write().await;
|
||||
@@ -944,6 +1075,7 @@ impl HealTask {
|
||||
|
||||
pub async fn cancel(&self) -> Result<()> {
|
||||
self.cancel_token.cancel();
|
||||
self.outcome.write().await.finish(Some(HealAbortReason::Cancelled));
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Cancelled;
|
||||
debug!(
|
||||
|
||||
@@ -214,6 +214,7 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
failed = failed.saturating_add(1);
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
if err.is_recoverable_heal() {
|
||||
retryable = retryable.saturating_add(1);
|
||||
} else {
|
||||
@@ -260,6 +261,7 @@ impl HealTask {
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let previous_progress = self.get_progress().await;
|
||||
let mut scanned = 0u64;
|
||||
let mut healed = 0u64;
|
||||
let mut failed = 0u64;
|
||||
@@ -304,23 +306,47 @@ impl HealTask {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
let mut listing_attempt = 0;
|
||||
let (objects, next_token, is_truncated) = loop {
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
};
|
||||
match page {
|
||||
Ok(page) => break page,
|
||||
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
|
||||
Err(error) => {
|
||||
self.outcome.write().await.attempt_failed();
|
||||
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
listing_attempt += 1;
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
self.outcome.write().await.mark_untraversable();
|
||||
return Err(Error::HealListingFailed {
|
||||
bucket: bucket.to_string(),
|
||||
source: Box::new(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut pending = objects;
|
||||
@@ -338,6 +364,14 @@ impl HealTask {
|
||||
self.check_control_flags().await?;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
@@ -380,7 +414,31 @@ impl HealTask {
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
};
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail: None,
|
||||
});
|
||||
return Err(err);
|
||||
}
|
||||
_ => self.outcome.write().await.attempt_failed(),
|
||||
}
|
||||
detail = Some(err.to_string());
|
||||
if Self::is_dangling_delete_grace_error(&err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -395,6 +453,10 @@ impl HealTask {
|
||||
"Heal bucket object dangling cleanup deferred by grace window"
|
||||
);
|
||||
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
disposition = HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientUsageCache,
|
||||
retry_not_before: None,
|
||||
};
|
||||
telemetry_unknown |= !increment_counter(&mut skipped);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -425,6 +487,11 @@ impl HealTask {
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() {
|
||||
HealFailureClass::RetryExhausted
|
||||
} else {
|
||||
HealFailureClass::Permanent
|
||||
});
|
||||
telemetry_unknown |= !increment_counter(&mut failed);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
@@ -459,8 +526,20 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(scanned, healed, failed, skipped, bytes);
|
||||
progress.update_object_progress(
|
||||
previous_progress.objects_scanned.saturating_add(scanned),
|
||||
previous_progress.objects_healed.saturating_add(healed),
|
||||
previous_progress.objects_failed.saturating_add(failed),
|
||||
previous_progress.skipped_objects.saturating_add(skipped),
|
||||
previous_progress.bytes_processed.saturating_add(bytes),
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
@@ -475,7 +554,7 @@ impl HealTask {
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated but no continuation token: end of listing.
|
||||
// Truncated without a continuation token is a compatibility EOF.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,8 +261,8 @@ impl HealTask {
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
read_repair: false,
|
||||
pool: None,
|
||||
set: None,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_result = self
|
||||
|
||||
@@ -14,6 +14,364 @@
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
|
||||
mod canonical_outcome {
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
|
||||
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
|
||||
HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_retries_only_the_failed_listing_page() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect("second-page retry succeeds");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 2);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a", "object-b"]
|
||||
);
|
||||
assert_eq!(
|
||||
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
|
||||
[None, Some("second".to_string()), Some("second".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_listing_page_cannot_restart_the_bucket() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(4)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect_err("listing page budget exhausted");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.counters.attempt_failures, 4);
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 1);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]);
|
||||
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_failure_preserves_processed_objects_and_partial_coverage() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
fail_second_listing_page: true,
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect_err("second page cannot be traversed");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.objects[0].identity.object, "object-a");
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_preserves_cumulative_progress_across_buckets() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
list_each_bucket: true,
|
||||
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
);
|
||||
task.execute().await.expect("both buckets complete");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 4);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4));
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.bucket == "bucket-b")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"object-a".to_string(),
|
||||
(0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(),
|
||||
);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("legacy adapter retains batch failure");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1));
|
||||
assert_eq!(outcome.counters.attempt_failures, 4);
|
||||
let failed = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("failed object");
|
||||
assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted));
|
||||
let object_b_calls = {
|
||||
let calls = storage.heal_object_calls.lock().expect("calls");
|
||||
calls.iter().filter(|object| object.as_str() == "object-b").count()
|
||||
};
|
||||
assert_eq!(object_b_calls, 1);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_success_counts_one_terminal_outcome() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect("retry should recover");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "object-a")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
outcome.counters.processed,
|
||||
outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]));
|
||||
let task = bucket_task(storage);
|
||||
task.execute().await.expect("grace permits traversal completion");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt");
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown);
|
||||
assert!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| item.identity.bucket_incarnation_id.is_none())
|
||||
);
|
||||
assert_eq!(
|
||||
task.get_progress().await.objects_healed,
|
||||
1,
|
||||
"legacy display count remains distinct from proof"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grace_single_object_is_completed_but_deferred() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage);
|
||||
task.execute().await.expect("grace is deferred");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_run_and_transient_existence_do_not_prove_repair() {
|
||||
for transient in [false, true] {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
if transient {
|
||||
storage
|
||||
.object_exists_by_name
|
||||
.lock()
|
||||
.expect("existence fixture")
|
||||
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
||||
}
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.dry_run = !transient;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute().await.expect("observation may complete");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
if transient {
|
||||
assert!(matches!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::TransientExistenceCheck,
|
||||
..
|
||||
}
|
||||
));
|
||||
} else {
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
|
||||
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.execute().await.expect_err("structural bucket error");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
|
||||
["bucket-a", "bucket-b"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_and_deadline_leave_partial_coverage() {
|
||||
for cancel in [false, true] {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_heal_object: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.timeout = Some(Duration::from_secs(1));
|
||||
let task = HealTask::from_request(request, storage);
|
||||
if cancel {
|
||||
task.cancel().await.expect("cancel request");
|
||||
}
|
||||
task.execute().await.expect_err("control interruption");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(
|
||||
outcome.execution,
|
||||
HealExecutionOutcome::Aborted(if cancel {
|
||||
HealAbortReason::Cancelled
|
||||
} else {
|
||||
HealAbortReason::Deadline
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decode_keeps_the_requested_pool_and_set() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string()));
|
||||
request.options.pool_index = Some(2);
|
||||
request.options.set_index = Some(3);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
task.execute().await.expect("decode fixture");
|
||||
let pool_and_set = {
|
||||
let options = storage.object_heal_opts.lock().expect("storage options");
|
||||
(options[0].pool, options[0].set)
|
||||
};
|
||||
assert_eq!(pool_and_set, (Some(2), Some(3)));
|
||||
let outcome = task.get_outcome().await;
|
||||
let identity = &outcome.objects[0].identity;
|
||||
assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3)));
|
||||
assert_eq!(identity.version_id.as_deref(), Some("version-a"));
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
|
||||
}
|
||||
}
|
||||
use crate::heal::storage::{HealListItem, HealObjectInfo};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||
@@ -582,6 +940,10 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
fail_second_listing_page: bool,
|
||||
recoverable_second_page_failures: Mutex<Option<usize>>,
|
||||
listing_tokens: Mutex<Vec<Option<String>>>,
|
||||
healed_objects: Mutex<Vec<String>>,
|
||||
heal_object_calls: Mutex<Vec<String>>,
|
||||
heal_object_version_ids: Mutex<Vec<Option<String>>>,
|
||||
@@ -995,12 +1357,41 @@ impl HealStorageAPI for MockStorage {
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
|
||||
self.listing_tokens
|
||||
.lock()
|
||||
.expect("listing tokens")
|
||||
.push(continuation_token.map(ToOwned::to_owned));
|
||||
if let Some(remaining) = self
|
||||
.recoverable_second_page_failures
|
||||
.lock()
|
||||
.expect("listing failures")
|
||||
.as_mut()
|
||||
{
|
||||
if continuation_token.is_none() {
|
||||
return Ok((vec![heal_item("object-a")], Some("second".to_string()), true));
|
||||
}
|
||||
if *remaining > 0 {
|
||||
*remaining -= 1;
|
||||
return Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
"page".to_string(),
|
||||
)));
|
||||
}
|
||||
return Ok((vec![heal_item("object-b")], None, false));
|
||||
}
|
||||
if self.fail_second_listing_page {
|
||||
return if continuation_token.is_none() {
|
||||
Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true))
|
||||
} else {
|
||||
Err(Error::other("listing unavailable"))
|
||||
};
|
||||
}
|
||||
if *self.truncate_without_token.lock().unwrap() {
|
||||
return Ok((vec![heal_item("object-a")], None, true));
|
||||
}
|
||||
|
||||
let mut listed = self.listed.lock().unwrap();
|
||||
if continuation_token.is_none() && !*listed {
|
||||
if continuation_token.is_none() && (!*listed || self.list_each_bucket) {
|
||||
*listed = true;
|
||||
let objects = if bucket == RUSTFS_META_BUCKET {
|
||||
vec![
|
||||
@@ -1393,6 +1784,8 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
|
||||
// A version listing can report the final page as truncated with no
|
||||
// continuation token. That is treated as end-of-listing (not an error),
|
||||
// so the returned page is healed and the pass terminates cleanly instead
|
||||
@@ -1414,10 +1807,16 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.heal_bucket("bucket-a")
|
||||
task.execute()
|
||||
.await
|
||||
.expect("truncated-without-token must terminate cleanly, not loop or error");
|
||||
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
|
||||
assert_eq!(
|
||||
storage.healed_objects.lock().unwrap().as_slice(),
|
||||
["object-a".to_string()],
|
||||
|
||||
@@ -1888,9 +1888,9 @@ where
|
||||
// A remote restart or movement flip invalidates
|
||||
// the token proof; usage_store interprets this
|
||||
// as a publication barrier and performs no PUT.
|
||||
return true;
|
||||
return Some(ScannerCycleDeferReason::DataMovement);
|
||||
}
|
||||
storeapi.scanner_data_usage_publication_blocked().await
|
||||
scanner_local_publication_defer_reason(storeapi.as_ref()).await
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -3239,8 +3239,8 @@ where
|
||||
{
|
||||
match status {
|
||||
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
|
||||
if storeapi.scanner_data_usage_publication_blocked().await {
|
||||
return Some(ScannerCycleDeferReason::DataMovement);
|
||||
if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await {
|
||||
return Some(reason);
|
||||
}
|
||||
if status == ScannerCycleStatus::Complete {
|
||||
let distributed = storeapi.setup_is_dist_erasure().await;
|
||||
@@ -3263,6 +3263,22 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_local_publication_defer_reason<S>(storeapi: &S) -> Option<ScannerCycleDeferReason>
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
if !storeapi.scanner_data_usage_publication_blocked().await {
|
||||
return None;
|
||||
}
|
||||
// Pending namespace commits invalidate this publication attempt, but only
|
||||
// storage movement creates durable, rate-limited catch-up debt.
|
||||
if storeapi.scanner_data_movement_pause_status().await.paused {
|
||||
Some(ScannerCycleDeferReason::DataMovement)
|
||||
} else {
|
||||
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_post_lease_activity_defer_reason(
|
||||
expected_digest: Option<[u8; 32]>,
|
||||
activity: Result<ScannerActivitySnapshot, String>,
|
||||
|
||||
@@ -266,6 +266,11 @@ async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
|
||||
}
|
||||
let pause_status = store.scanner_data_movement_pause_status().await;
|
||||
assert!(pause_status.paused);
|
||||
assert_eq!(
|
||||
scanner_local_publication_defer_reason(store.as_ref()).await,
|
||||
Some(ScannerCycleDeferReason::DataMovement),
|
||||
"an actual data-movement pause must retain durable catch-up tracking"
|
||||
);
|
||||
paused_probe.wait().await;
|
||||
drop(paused_probe);
|
||||
|
||||
@@ -1171,6 +1176,9 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now())
|
||||
.await
|
||||
.expect("scanner pause backlog should be available");
|
||||
let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default())
|
||||
@@ -1195,6 +1203,13 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
.await
|
||||
.expect("fixture usage baseline should be readable");
|
||||
let pending = ecstore_hold_namespace_commit(store.as_ref());
|
||||
assert_eq!(
|
||||
scanner_local_publication_defer_reason(store.as_ref()).await,
|
||||
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
"an ordinary namespace commit must not be classified as data movement"
|
||||
);
|
||||
let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
|
||||
assert_eq!(pause_backlog_attempt, ScannerPauseBacklogAttemptDecision::Untracked);
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let mut cycle_info = CurrentCycle {
|
||||
@@ -1209,7 +1224,15 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
|
||||
.await
|
||||
.expect("the coordinator must finish its namespace walk while a PUT is pending");
|
||||
assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal");
|
||||
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert_eq!(
|
||||
outcome,
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
);
|
||||
finish_scanner_pause_backlog_cycle(&mut pause_backlog, &store, pause_backlog_attempt, outcome).await;
|
||||
let pause_backlog_status = scanner_pause_backlog_status(store.clone()).await;
|
||||
assert_eq!(pause_backlog_status.phase, ScannerPauseBacklogPhase::Idle);
|
||||
assert!(!pause_backlog_status.pending_full_scan);
|
||||
assert_eq!(pause_backlog_status.catch_up_attempts, 0);
|
||||
assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle");
|
||||
assert_eq!(revision, DataUsageCacheRevision::Missing);
|
||||
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
|
||||
@@ -5826,7 +5849,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier
|
||||
let probe_calls = route_probe_calls.clone();
|
||||
async move {
|
||||
let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
route_blocked && call > 1
|
||||
(route_blocked && call > 1).then_some(ScannerCycleDeferReason::DataMovement)
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -5872,16 +5895,19 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
|
||||
data: None,
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
|| async { true },
|
||||
|| async { Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) },
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert_eq!(
|
||||
outcome,
|
||||
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
);
|
||||
assert!(!store.objects.lock().await.contains_key(&target_key));
|
||||
assert_eq!(
|
||||
store.put_counts.lock().await.get(&target_key),
|
||||
None,
|
||||
"the final pool-state fence must run before the first PUT"
|
||||
"the final publication fence must run before the first PUT"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5902,7 +5928,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing(
|
||||
receiver,
|
||||
None,
|
||||
None,
|
||||
|| async { false },
|
||||
|| async { None },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -5950,7 +5976,7 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden
|
||||
receiver,
|
||||
None,
|
||||
None,
|
||||
|| async { false },
|
||||
|| async { None },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -5991,7 +6017,7 @@ async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() {
|
||||
receiver,
|
||||
None,
|
||||
None,
|
||||
|| async { false },
|
||||
|| async { None },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -6051,7 +6077,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
|
||||
data: Some(Bytes::from(snapshot_data)),
|
||||
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
|
||||
}),
|
||||
|| async { true },
|
||||
|| async { Some(ScannerCycleDeferReason::DataMovement) },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -6091,7 +6117,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() {
|
||||
// Model the remote lease holder flipping its movement generation
|
||||
// after the activity probe but before the coordinator's PUT.
|
||||
route_store.publication_admission_blocked.store(true, Ordering::Release);
|
||||
false
|
||||
None
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -6129,7 +6155,7 @@ async fn coordinator_classifies_an_expired_publication_lease() {
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
ScannerPublicationFence::new(None, Some(expired), None),
|
||||
|| async { false },
|
||||
|| async { None },
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -6208,7 +6234,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() {
|
||||
data: None,
|
||||
revision: DataUsageCacheRevision::Missing,
|
||||
}),
|
||||
|| async { true },
|
||||
|| async { Some(ScannerCycleDeferReason::DataMovement) },
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -265,7 +265,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
receiver,
|
||||
leader_epoch,
|
||||
initial_baseline,
|
||||
|| async { false },
|
||||
|| async { None },
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -280,7 +280,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
|
||||
{
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
|
||||
ctx,
|
||||
@@ -308,7 +308,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
|
||||
{
|
||||
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
|
||||
ctx,
|
||||
@@ -336,7 +336,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
|
||||
) -> DataUsagePersistOutcome
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync,
|
||||
Fut: Future<Output = bool> + Send,
|
||||
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
|
||||
{
|
||||
let ScannerPublicationFence {
|
||||
expected_publication_epoch,
|
||||
@@ -374,18 +374,19 @@ where
|
||||
} else {
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str()
|
||||
};
|
||||
if route_probe().await {
|
||||
if let Some(reason) = route_probe().await {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_blocked_before_reconcile",
|
||||
"Scanner data usage publication deferred by the pool-state fence"
|
||||
reason = reason.as_str(),
|
||||
path = %target_path,
|
||||
"Scanner data usage publication deferred by the publication fence"
|
||||
);
|
||||
global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str());
|
||||
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
global_metrics().record_scanner_usage_deferred(reason.as_str());
|
||||
outcome = DataUsagePersistOutcome::Deferred(reason);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -626,17 +627,18 @@ where
|
||||
if ctx.is_cancelled() {
|
||||
break 'updates;
|
||||
}
|
||||
if route_probe().await {
|
||||
if let Some(reason) = route_probe().await {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_blocked_before_save",
|
||||
"Scanner data usage publication deferred by the final pool-state fence"
|
||||
reason = reason.as_str(),
|
||||
path = %target_path,
|
||||
"Scanner data usage publication deferred by the final publication fence"
|
||||
);
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break DataUsagePersistOutcome::Deferred(reason);
|
||||
}
|
||||
if remote_lease_expired(remote_lease_deadline) {
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
|
||||
@@ -722,19 +724,19 @@ where
|
||||
);
|
||||
}
|
||||
Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
|
||||
let route_blocked = route_probe().await;
|
||||
if route_blocked {
|
||||
if let Some(reason) = route_probe().await {
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %target_path,
|
||||
state = "publication_deferred",
|
||||
reason = reason.as_str(),
|
||||
path = %target_path,
|
||||
error = %e,
|
||||
"Scanner data usage route is blocked by data movement; retrying later"
|
||||
"Scanner data usage route remains blocked; retrying later"
|
||||
);
|
||||
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
|
||||
break DataUsagePersistOutcome::Deferred(reason);
|
||||
}
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
|
||||
@@ -1294,6 +1294,8 @@ impl FolderScanner {
|
||||
}
|
||||
Err(e) => return Err(ScannerError::Io(e)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
|
||||
@@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
|
||||
@@ -20,6 +20,8 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
mod segment_observation;
|
||||
|
||||
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
|
||||
const STATIC_OBJECTS: u64 = 24;
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
//! Fixture-only range diagnostics. No result is supplied to a scan selector.
|
||||
|
||||
use super::*;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
const MAX_SEGMENTS: usize = 4;
|
||||
const MAX_SEGMENT_BYTES: usize = 128;
|
||||
const MAX_WALK_SAMPLES: usize = 32;
|
||||
const MAX_WALK_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum ProposalError {
|
||||
EntryLimit,
|
||||
ByteLimit,
|
||||
InvalidKey,
|
||||
}
|
||||
|
||||
// Keys come from successful fixture writes, not a production mutation stream.
|
||||
fn fixture_proposal(keys: &[&str]) -> Result<BTreeSet<String>, ProposalError> {
|
||||
let mut segments = BTreeSet::new();
|
||||
let mut bytes = 0;
|
||||
for key in keys {
|
||||
if key.is_empty() || key.contains(['\\', '\0']) || key.split('/').any(|part| matches!(part, "" | "." | "..")) {
|
||||
return Err(ProposalError::InvalidKey);
|
||||
}
|
||||
let segment = key.split('/').next().expect("validated nonempty key");
|
||||
if segments.contains(segment) {
|
||||
continue;
|
||||
}
|
||||
if segments.len() == MAX_SEGMENTS {
|
||||
return Err(ProposalError::EntryLimit);
|
||||
}
|
||||
if segment.len() > MAX_SEGMENT_BYTES - bytes {
|
||||
return Err(ProposalError::ByteLimit);
|
||||
}
|
||||
bytes += segment.len();
|
||||
segments.insert(segment.to_string());
|
||||
}
|
||||
Ok(segments)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_observation_fixture_proposal_bounds() {
|
||||
assert_eq!(fixture_proposal(&["hot/one", "hot/two"]), Ok(BTreeSet::from(["hot".to_string()])));
|
||||
assert_eq!(fixture_proposal(&["a", "b", "c", "d"]).expect("entry boundary").len(), MAX_SEGMENTS);
|
||||
assert_eq!(fixture_proposal(&["a", "b", "c", "d", "e"]), Err(ProposalError::EntryLimit));
|
||||
let exact = "x".repeat(MAX_SEGMENT_BYTES);
|
||||
assert!(fixture_proposal(&[&exact]).is_ok());
|
||||
assert_eq!(fixture_proposal(&[&exact, "y"]), Err(ProposalError::ByteLimit));
|
||||
let oversized = "x".repeat(MAX_SEGMENT_BYTES + 1);
|
||||
assert_eq!(fixture_proposal(&[&oversized]), Err(ProposalError::ByteLimit));
|
||||
for key in ["", "/hot", "hot/../cold", "hot//one", "hot\\one", "hot/\0"] {
|
||||
assert_eq!(fixture_proposal(&[key]), Err(ProposalError::InvalidKey));
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_value(cache: &DataUsageCache) -> serde_json::Value {
|
||||
let mut value = serde_json::to_value(cache).expect("serialize the entire cache");
|
||||
// Children are a HashSet: canonicalize only that unordered field, without
|
||||
// discarding any cache fields or changing ordered histogram arrays.
|
||||
for (path, entry) in &cache.cache {
|
||||
value["cache"][path]["children"] =
|
||||
serde_json::to_value(entry.children.iter().collect::<BTreeSet<_>>()).expect("canonical child set");
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for prefix in ["hot", "cold", "other"] {
|
||||
for leaf in ["one", "two"] {
|
||||
let object = format!("{prefix}/{leaf}");
|
||||
let mut metadata = FileMeta::new();
|
||||
let mut info = FileInfo::new(&object, 4, 2);
|
||||
info.volume = "bucket".to_string();
|
||||
info.name = object.clone();
|
||||
info.size = 1;
|
||||
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
|
||||
info.metadata.insert("etag".to_string(), "before".to_string());
|
||||
metadata.add_version(info).expect("construct segment fixture metadata");
|
||||
write_test_object_metadata_bytes(&root, "bucket", &object, &metadata.marshal_msg().expect("encode metadata")).await;
|
||||
}
|
||||
}
|
||||
let changed_key = "hot/one";
|
||||
let changed_path = root.join("bucket").join(changed_key).join("xl.meta");
|
||||
let before = tokio::fs::read(&changed_path).await.expect("read initial hot metadata");
|
||||
let mut metadata = FileMeta::new();
|
||||
let mut info = FileInfo::new(changed_key, 4, 2);
|
||||
info.volume = "bucket".to_string();
|
||||
info.name = changed_key.to_string();
|
||||
info.size = 1;
|
||||
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
|
||||
info.metadata.insert("etag".to_string(), "after!".to_string());
|
||||
metadata.add_version(info).expect("construct same-size hot mutation");
|
||||
write_test_object_metadata_bytes(&root, "bucket", changed_key, &metadata.marshal_msg().expect("encode hot mutation")).await;
|
||||
let after = tokio::fs::read(&changed_path)
|
||||
.await
|
||||
.expect("read back committed fixture mutation");
|
||||
assert_eq!(before.len(), after.len(), "fixture rewrite must keep metadata byte length unchanged");
|
||||
assert_ne!(before, after, "a changed key requires an observable successful fixture write");
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
let paths = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let proposed_walked = Arc::new(Mutex::new(BTreeSet::<String>::new()));
|
||||
scanner.update_current_path = Arc::new({
|
||||
let paths = paths.clone();
|
||||
let proposed_walked = proposed_walked.clone();
|
||||
move |path: &str| {
|
||||
let mut paths = paths.lock().expect("lock bounded actual-walk samples");
|
||||
assert!(paths.len() < MAX_WALK_SAMPLES, "fixture walk exceeded its entry budget");
|
||||
let bytes: usize = paths.iter().map(String::len).sum();
|
||||
assert!(path.len() <= MAX_WALK_BYTES - bytes, "fixture walk exceeded its byte budget");
|
||||
paths.push(path.to_string());
|
||||
if observe {
|
||||
let proposed = fixture_proposal(&[changed_key]).expect("bounded successful fixture mutation");
|
||||
if let Some(segment) = path.strip_prefix("bucket/").and_then(|path| path.split('/').next())
|
||||
&& proposed.contains(segment)
|
||||
{
|
||||
proposed_walked
|
||||
.lock()
|
||||
.expect("lock bounded observed segments")
|
||||
.insert(segment.to_string());
|
||||
}
|
||||
}
|
||||
Box::pin(async {})
|
||||
}
|
||||
});
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await
|
||||
.expect("actual folder walker must finish independently of diagnostics");
|
||||
let paths = paths.lock().expect("read walk samples").clone();
|
||||
assert!(!paths.is_empty());
|
||||
for prefix in ["hot", "cold", "other"] {
|
||||
assert!(
|
||||
paths.iter().any(|path| path == &format!("bucket/{prefix}")),
|
||||
"all fixture segments must actually be walked"
|
||||
);
|
||||
}
|
||||
let store = FixtureStore::new();
|
||||
let revisions = DataUsageCache::default()
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("read empty fixture revisions");
|
||||
scanner
|
||||
.new_cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save actual walker output through the cache codec and revision gate");
|
||||
let loaded = store.strict_load().await;
|
||||
assert_eq!(loaded.checked_flatten("bucket").expect("complete fixture tree").objects, 6);
|
||||
assert_eq!(
|
||||
cache_value(&loaded),
|
||||
cache_value(&scanner.new_cache),
|
||||
"codec round-trip must retain the entire cache, not just aggregate size"
|
||||
);
|
||||
if observe {
|
||||
let proposed = proposed_walked.lock().expect("read callback observations").clone();
|
||||
assert_eq!(proposed, BTreeSet::from(["hot".to_string()]));
|
||||
let walked_segments: BTreeSet<_> = paths
|
||||
.iter()
|
||||
.filter_map(|path| path.strip_prefix("bucket/"))
|
||||
.filter_map(|path| path.split('/').next())
|
||||
.collect();
|
||||
assert_eq!(walked_segments, BTreeSet::from(["cold", "hot", "other"]));
|
||||
assert!(proposed.iter().all(|segment| walked_segments.contains(segment.as_str())));
|
||||
assert_eq!(
|
||||
walked_segments.len() - proposed.len(),
|
||||
2,
|
||||
"the two non-proposed segments must still be walked"
|
||||
);
|
||||
eprintln!(
|
||||
"segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified",
|
||||
paths.len()
|
||||
);
|
||||
} else {
|
||||
assert!(proposed_walked.lock().expect("read disabled observations").is_empty());
|
||||
}
|
||||
// Compare semantic values because map encoding order is not content identity.
|
||||
(paths, cache_value(&loaded))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn segment_observation_on_off_preserves_actual_walk_and_saved_cache() {
|
||||
let off = walk_and_save(false).await;
|
||||
let on = walk_and_save(true).await;
|
||||
assert_eq!(off.0, on.0, "diagnostics must not change actual traversal order or coverage");
|
||||
assert_eq!(off.1, on.1, "diagnostics must not change the saved cache result");
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST";
|
||||
|
||||
struct Observation {
|
||||
root: PathBuf,
|
||||
limit: u64,
|
||||
entries: u64,
|
||||
name_bytes: u64,
|
||||
}
|
||||
|
||||
static OBSERVATION: Mutex<Option<Observation>> = Mutex::new(None);
|
||||
|
||||
// Only the selected synthetic disk is observed; concurrent unrelated scanners
|
||||
// do not consume its budget. This hook is absent from non-test builds.
|
||||
pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) {
|
||||
let mut guard = OBSERVATION.lock().expect("enumeration observation lock");
|
||||
if let Some(observation) = guard.as_mut()
|
||||
&& Path::new(dir).starts_with(&observation.root)
|
||||
{
|
||||
observation.entries += 1;
|
||||
observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name");
|
||||
if observation.entries >= observation.limit {
|
||||
budget.cancel_for_runtime();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationGuard;
|
||||
|
||||
impl Drop for ObservationGuard {
|
||||
fn drop(&mut self) {
|
||||
*OBSERVATION.lock().expect("enumeration observation cleanup") = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Request {
|
||||
workspace: PathBuf,
|
||||
objects: usize,
|
||||
raw_entry_budget: u64,
|
||||
round: u32,
|
||||
}
|
||||
|
||||
async fn read_bounded(path: &Path) -> Vec<u8> {
|
||||
let file = tokio::fs::File::open(path).await.expect("open fixture artifact");
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_CACHE_BYTES + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.expect("read fixture artifact");
|
||||
assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES);
|
||||
bytes
|
||||
}
|
||||
|
||||
async fn round(request: &Request) -> serde_json::Value {
|
||||
assert!((1..=1024).contains(&request.objects));
|
||||
assert!((1..=4096).contains(&request.raw_entry_budget));
|
||||
assert!(request.round < 64);
|
||||
let disk_root = request.workspace.join("disk");
|
||||
let cache_path = request.workspace.join("cache.bin");
|
||||
if request.round == 0 {
|
||||
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..request.objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
initial.info.skip_healing = true;
|
||||
initial.info.snapshot_complete = false;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
|
||||
.await
|
||||
.expect("persist initial cache");
|
||||
}
|
||||
let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan");
|
||||
assert_eq!(cache.info.name, "bucket");
|
||||
let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects;
|
||||
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open synthetic disk in this process");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
*OBSERVATION.lock().expect("install observation") = Some(Observation {
|
||||
root: disk.path(),
|
||||
limit: request.raw_entry_budget,
|
||||
entries: 0,
|
||||
name_bytes: 0,
|
||||
});
|
||||
let _observation_guard = ObservationGuard;
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk,
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
let (returned, outcome) = match result {
|
||||
Ok(cache) => (cache, "complete"),
|
||||
Err(ScannerError::PartialCache(cache)) => (*cache, "partial"),
|
||||
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {
|
||||
(cache, "cancelled_without_cache")
|
||||
}
|
||||
Err(error) => panic!("unexpected real scanner failure: {error}"),
|
||||
};
|
||||
let encoded = returned.marshal_msg().expect("returned cache codec");
|
||||
assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES);
|
||||
tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache");
|
||||
let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec");
|
||||
let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root");
|
||||
let scanned = returned.checked_flatten("bucket").expect("returned bucket root");
|
||||
assert_eq!(
|
||||
(retained.objects, retained.versions, retained.size),
|
||||
(scanned.objects, scanned.versions, scanned.size)
|
||||
);
|
||||
assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete);
|
||||
let guard = OBSERVATION.lock().expect("read observation");
|
||||
let observation = guard.as_ref().expect("installed observation");
|
||||
serde_json::json!({
|
||||
"schema": 1, "pid": std::process::id(), "round": request.round,
|
||||
"objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget,
|
||||
"raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes,
|
||||
"objects_processed": budget.progress().0,
|
||||
"objects_before": before, "objects_retained": retained.objects,
|
||||
"versions_retained": retained.versions, "bytes_retained": retained.size,
|
||||
"snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome,
|
||||
})
|
||||
}
|
||||
|
||||
/// Default CI is a positive healthy control. The external driver selects the
|
||||
/// same worker in a fresh OS process per round and applies its strict oracle.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn enumeration_restart_worker() {
|
||||
if let Some(path) = std::env::var_os(REQUEST_ENV) {
|
||||
let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request");
|
||||
let report = round(&request).await;
|
||||
tokio::fs::write(
|
||||
request.workspace.join(format!("round-{}.json", request.round)),
|
||||
serde_json::to_vec(&report).expect("report JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("write worker report");
|
||||
} else {
|
||||
let temp = tempfile::tempdir().expect("healthy fixture directory");
|
||||
let report = round(&Request {
|
||||
workspace: temp.path().to_path_buf(),
|
||||
objects: 4,
|
||||
raw_entry_budget: 16,
|
||||
round: 0,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report["outcome"], "complete");
|
||||
assert_eq!(report["snapshot_complete"], true);
|
||||
assert_eq!(report["objects_retained"], 4);
|
||||
assert_eq!(report["versions_retained"], 4);
|
||||
assert_eq!(report["bytes_retained"], 4);
|
||||
assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}");
|
||||
}
|
||||
}
|
||||
@@ -124,6 +124,16 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
async fn wait_for_namespace_commit_tails(store: &ECStore) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while store.scanner_data_usage_publication_blocked().await {
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("namespace commit tails should drain before the scanner fixture runs");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_bucket_identity_uses_its_set_instance_owner() {
|
||||
@@ -334,6 +344,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work()
|
||||
.await
|
||||
.expect("initial object should persist");
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let mut baseline = None;
|
||||
for (index, (scan_mode, requires_full_scan, explicit_scope)) in [
|
||||
(HealScanMode::Normal, true, false),
|
||||
@@ -352,6 +363,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work()
|
||||
.put_object("cold-bucket", &format!("added-{index}"), &mut reader, &ScannerObjectOptions::default())
|
||||
.await
|
||||
.expect("cold bucket mutation should persist");
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
// Only the hot bucket is in the usage hint. The cold result must
|
||||
// come from this cycle's storage walk, not its previous baseline.
|
||||
record_dirty_usage_bucket("hot-bucket");
|
||||
|
||||
Reference in New Issue
Block a user