chore(test): merge main before startup CAS coverage

This commit is contained in:
overtrue
2026-09-06 12:45:50 +08:00
64 changed files with 5733 additions and 582 deletions
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2024 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::path::Path;
use std::process::Command;
fn git(root: &Path, args: &[&str]) -> Option<String> {
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
output
.status
.success()
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
}
fn emit(name: &str, value: &str) {
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
}
fn main() {
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
let root = Path::new(&manifest).join("../..");
// Cover dependency/common sources as well as this crate. HEAD/ref/index
// changes must refresh identity even when no Rust source mtime changes.
for path in [
"crates",
"rustfs",
"Cargo.toml",
"Cargo.lock",
"rust-toolchain.toml",
".cargo",
".config",
] {
println!("cargo:rerun-if-changed={}", root.join(path).display());
}
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
git_paths.push(reference);
}
for path in git_paths {
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
let path = Path::new(&path);
let path = if path.is_absolute() {
path.to_owned()
} else {
root.join(path)
};
if path.exists() {
println!("cargo:rerun-if-changed={}", path.display());
}
}
}
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
let mut features = std::env::vars()
.filter_map(|(key, _)| {
key.strip_prefix("CARGO_FEATURE_")
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
})
.collect::<Vec<_>>();
features.sort();
emit("COMMIT", &revision);
emit("DIRTY", if dirty { "true" } else { "false" });
emit("LOCK", &lock);
emit("FEATURES", &features.join(","));
for name in ["TARGET", "PROFILE"] {
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
}
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
emit("RUSTFLAGS_HEX", &flags);
}
+13 -3
View File
@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
/// tests need this lower-level record to prove that the rebuilt target holds the
/// `xl.meta` selected for a specific version and every `part.N` it declares.
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
pub(crate) struct VersionShardCensus {
pub version_id: Option<String>,
pub has_xl_meta: bool,
pub data_dir: Option<String>,
pub erasure_index: Option<usize>,
pub data_blocks: Option<usize>,
pub parity_blocks: Option<usize>,
pub expected_part_numbers: BTreeSet<usize>,
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
pub inline_data_fingerprint: Option<PartShardFingerprint>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
pub(crate) struct PartShardFingerprint {
pub size: u64,
pub sha256: String,
@@ -88,13 +90,15 @@ impl VersionShardCensus {
&& manifest.is_complete()
&& self.data_dir == manifest.data_dir
&& self.erasure_index == manifest.erasure_index
&& self.data_blocks == manifest.data_blocks
&& self.parity_blocks == manifest.parity_blocks
&& self.expected_part_numbers == manifest.expected_part_numbers
&& self.present_part_fingerprints == manifest.present_part_fingerprints
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
}
}
fn sha256_hex(data: &[u8]) -> String {
pub(crate) fn sha256_hex(data: &[u8]) -> String {
let digest = Sha256::digest(data);
digest.iter().map(|byte| format!("{byte:02x}")).collect()
}
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
has_xl_meta: false,
data_dir: None,
erasure_index: None,
data_blocks: None,
parity_blocks: None,
expected_part_numbers: BTreeSet::new(),
present_part_fingerprints: BTreeMap::new(),
inline_data_fingerprint: None,
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
has_xl_meta: true,
data_dir,
erasure_index,
data_blocks: Some(file_info.erasure.data_blocks),
parity_blocks: Some(file_info.erasure.parity_blocks),
expected_part_numbers,
present_part_fingerprints,
inline_data_fingerprint,
@@ -413,6 +421,8 @@ mod tests {
has_xl_meta: true,
data_dir: Some("data-dir".to_string()),
erasure_index: Some(3),
data_blocks: Some(2),
parity_blocks: Some(2),
expected_part_numbers: BTreeSet::from([1]),
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
inline_data_fingerprint: None,
+6 -2
View File
@@ -35,11 +35,15 @@ where
{
let mut last_usage = DataUsageInfo::default();
let mut last_query_error = None;
for _ in 0..45 {
for _ in 0..90 {
match get_data_usage_info(env).await {
Ok(usage) => {
last_query_error = None;
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
if usage.is_complete_bucket_usage_snapshot()
&& usage.usage_snapshot_converged != Some(false)
&& usage.buckets_usage.contains_key(bucket)
&& predicate(&usage)
{
return Ok(usage);
}
last_usage = usage;
@@ -16,15 +16,18 @@
#[cfg(test)]
mod tests {
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
use crate::common::{
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
rustfs_binary_path,
};
use crate::storage_api::RUSTFS_META_BUCKET;
use aws_sdk_s3::primitives::ByteStream;
use http::Method;
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::Command;
@@ -34,6 +37,76 @@ mod tests {
const POOL_METADATA_OBJECT: &str = "pool.bin";
#[derive(serde::Deserialize)]
struct EvidenceBuild {
sha256: String,
}
#[derive(serde::Deserialize)]
struct RestartEvidenceRun {
schema: u32,
run_id: String,
source_revision: String,
test_build: serde_json::Value,
binary: EvidenceBuild,
test_binary: EvidenceBuild,
}
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
let mut file = std::fs::File::open(path)?;
let mut digest = Sha256::new();
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
digest.update(&buffer[..read]);
}
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
}
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
return Ok(None);
};
let directory = PathBuf::from(directory);
let receipt = directory.join("run.json");
if receipt.metadata()?.len() > 1024 * 1024 {
return Err("oversized scanner/heal execution receipt".into());
}
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
return Err("invalid scanner/heal execution identity".into());
}
let built = compiled_test_identity();
for key in ["source_revision", "dirty", "lock_blob", "features"] {
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
}
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
assert_eq!(
file_sha256(&std::env::current_exe()?)?,
run.test_binary.sha256,
"test executable must match the run receipt"
);
if directory.join("background-target-restart.json").exists() {
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
}
Ok(Some((directory, run)))
}
fn compiled_test_identity() -> serde_json::Value {
serde_json::json!({
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
})
}
struct TcpPortBlackhole {
port: u16,
comment: String,
@@ -195,8 +268,9 @@ mod tests {
clients: &[aws_sdk_s3::Client],
bucket: &str,
expected_keys: &HashSet<String>,
) -> Result<(), Box<dyn Error + Send + Sync>> {
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
const PAGE_SIZE: i32 = 10;
let mut node_listings = Vec::with_capacity(clients.len());
for (node_index, client) in clients.iter().enumerate() {
let mut listed_keys = Vec::new();
let mut continuation_token = None;
@@ -243,8 +317,10 @@ mod tests {
&listed_key_set, expected_keys,
"node {node_index} did not expose the complete recovered namespace"
);
listed_keys.sort();
node_listings.push(listed_keys);
}
Ok(())
Ok(node_listings)
}
fn heal_task_status_diagnostic(body: &str) -> String {
@@ -808,6 +884,13 @@ mod tests {
}
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
restart_evidence_run(&server_binary)?
} else {
None
};
let mut evidence_objects = Vec::new();
let (background_enabled, interruption_node, interruption_kind) = match scenario {
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
@@ -855,7 +938,7 @@ mod tests {
for node_index in 0..cluster.nodes.len() {
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
}
cluster.start().await?;
cluster.start_with_binary(&server_binary).await?;
let clients = cluster.create_all_clients()?;
let bucket = "heal-restart-during-rebuild";
@@ -996,7 +1079,7 @@ mod tests {
}
}
cluster.start_node(1).await?;
cluster.start_node_from_binary(1, &server_binary).await?;
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1274,7 +1357,7 @@ mod tests {
}
}
}
cluster.start_node(interruption_node).await?;
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
if interruption_node == 0 {
let target = cluster.nodes[1]
.process
@@ -1373,7 +1456,7 @@ mod tests {
.map(|manifest| manifest.key.clone())
.collect::<HashSet<_>>();
assert!(expected_keys.insert(outage_key.to_string()));
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
let target_client = cluster.create_s3_client(1)?;
for expected in &expected_manifests {
@@ -1381,11 +1464,31 @@ mod tests {
let actual = response.body.collect().await?.into_bytes();
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
if evidence_run.is_some() {
evidence_objects.push(serde_json::json!({
"key": expected.key, "version_id": expected.shard_census.version_id,
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
"expected_sha256": sha256_hex(&expected_body),
"actual_sha256": sha256_hex(&actual),
"expected_physical": expected.shard_census,
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
}));
}
}
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
let actual = response.body.collect().await?.into_bytes();
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
if evidence_run.is_some() {
evidence_objects.push(serde_json::json!({
"key": outage_key, "version_id": null,
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
"expected_sha256": sha256_hex(&expected_outage_body),
"actual_sha256": sha256_hex(&actual),
"expected_physical": null,
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
}));
}
let terminal_deadline = Instant::now() + Duration::from_secs(30);
loop {
@@ -1432,6 +1535,31 @@ mod tests {
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
}
if let Some((directory, run)) = evidence_run {
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
assert_ne!(target_pid, restarted_pid, "target must be a new process");
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
let evidence = serde_json::json!({
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
"run_id": run.run_id, "source_revision": run.source_revision,
"test_build": compiled_test_identity(),
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
"pid_before": target_pid, "pid_after": restarted_pid,
"objects": evidence_objects, "node_listings": node_listings,
});
let data = serde_json::to_vec(&evidence)?;
if data.len() > 1024 * 1024 {
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
}
let mut output = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(directory.join("background-target-restart.json"))?;
output.write_all(&data)?;
output.sync_all()?;
}
Ok(())
}
+1 -1
View File
@@ -32,7 +32,7 @@ pub mod bucket {
pub mod bucket_target_sys {
pub use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query,
SsecPassthroughCapability, TargetClient, append_version_id_query,
};
}
+38 -106
View File
@@ -369,26 +369,6 @@ struct SsecPassthroughRecord {
recorded_at: Instant,
}
/// What a target write does when the bucket's persisted target set exists but
/// cannot be decoded.
///
/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a
/// configuration that could not be fully read, because re-serializing a
/// partial in-memory view is the one mechanism by which a configured target
/// really disappears. That rule guards against an *unintentional* overwrite,
/// so an operator who names the hazard keeps a repair path
/// (rustfs/backlog#2309); everything that does not name it stays refused.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum UnreadableTargetsPolicy {
/// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`].
#[default]
FailClosed,
/// Discard the unreadable set; the target being written becomes the whole
/// configuration. Reachable only from an admin request that asked for it
/// explicitly, and audited by the caller.
Replace,
}
#[derive(Debug, Default)]
pub struct BucketTargetSys {
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
@@ -811,41 +791,23 @@ impl BucketTargetSys {
bucket: &str,
target: &BucketTarget,
update: bool,
unreadable_policy: UnreadableTargetsPolicy,
) -> Result<BucketTargets, BucketTargetError> {
self.validate_target(bucket, target).await?;
let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?;
let mut bucket_targets = self.targets_base_for_write(bucket).await?;
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
Ok(bucket_targets)
}
/// The persisted target set a write merges into.
///
/// An absent configuration starts from the empty set. An unreadable one is
/// refused, because re-serializing a partial view of a set this node could
/// not decode is how a configured target disappears for good — unless the
/// caller carries the operator's explicit
/// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it
/// deliberately (rustfs/backlog#2309).
async fn targets_base_for_write(
&self,
bucket: &str,
unreadable_policy: UnreadableTargetsPolicy,
) -> Result<BucketTargets, BucketTargetError> {
/// Ordinary writes must not turn an unreadable cached snapshot into an
/// empty configuration. Explicit repair belongs to the metadata transaction
/// that can inspect the current persisted state.
async fn targets_base_for_write(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
match self.list_bucket_targets(bucket).await {
Ok(targets) => Ok(targets),
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
// The opt-in discards only a set this node genuinely cannot read.
// A readable set still merges through the arm above, so the policy
// can never drop a target that was visible here.
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
if unreadable_policy == UnreadableTargetsPolicy::Replace =>
{
Ok(BucketTargets::default())
}
Err(err) => Err(err),
}
}
@@ -908,7 +870,9 @@ impl BucketTargetSys {
Ok(())
}
fn upsert_target_entry(
/// Merge a validated target into a caller-owned snapshot. The caller must
/// protect that snapshot through persistence.
pub fn upsert_target_entry(
bucket_targets: &mut Vec<BucketTarget>,
target: &BucketTarget,
update: bool,
@@ -1272,25 +1236,27 @@ impl BucketTargetSys {
return (String::new(), false);
};
{
let targets_map = self.targets_map.read().await;
if let Some(targets) = targets_map.get(bucket) {
for tgt in targets {
if tgt.target_type == target.target_type
&& tgt.target_bucket == target.target_bucket
&& target.endpoint == tgt.endpoint
&& tgt
.credentials
.as_ref()
.map(|c| {
let default_creds = Credentials::default();
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
})
.unwrap_or(false)
{
return (tgt.arn.clone(), true);
}
}
let targets_map = self.targets_map.read().await;
let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default();
Self::remote_arn_for_targets(targets, target, depl_id)
}
/// Resolve create idempotency against the snapshot the caller will persist.
pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) {
for tgt in targets {
if tgt.target_type == target.target_type
&& tgt.target_bucket == target.target_bucket
&& target.endpoint == tgt.endpoint
&& tgt
.credentials
.as_ref()
.map(|c| {
let default_creds = Credentials::default();
c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
})
.unwrap_or(false)
{
return (tgt.arn.clone(), true);
}
}
@@ -4371,47 +4337,19 @@ mod tests {
}
}
/// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable
/// `bucket-targets.json` left the bucket with no API repair path at all.
/// The refusal is the default and stays the default; the operator's
/// explicit opt-in is the only thing that discards the set, and it starts
/// the replacement from empty rather than from a partial view of bytes
/// this node never decoded.
#[tokio::test]
async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() {
async fn an_unreadable_target_set_refuses_cached_writes() {
let sys = BucketTargetSys::default();
let bucket = "targets-repair-opt-in";
sys.mark_targets_unreadable(bucket).await;
assert!(
matches!(
sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
),
"without the opt-in an unreadable target set must still refuse the write"
);
assert_eq!(
UnreadableTargetsPolicy::default(),
UnreadableTargetsPolicy::FailClosed,
"a caller that says nothing must get the refusal"
);
let base = sys
.targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace)
.await
.expect("the explicit opt-in must let an operator replace an unreadable set");
assert!(
base.is_empty(),
"the replacement must start from an empty set, never from a partial decode"
);
assert!(matches!(
sys.targets_base_for_write(bucket).await,
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
));
}
/// The opt-in is not a wipe switch. On a set this node can read, both
/// policies take the same merge path, so a stray `replace-unreadable=true`
/// cannot drop a visible target — which is what makes the flag safe to
/// repeat in an operator's repair script.
#[tokio::test]
async fn the_opt_in_never_discards_a_readable_target_set() {
async fn a_readable_target_set_remains_the_write_base() {
let sys = BucketTargetSys::default();
let bucket = "targets-repair-readable";
let existing = repair_target(bucket, "keep");
@@ -4419,14 +4357,8 @@ mod tests {
.write()
.await
.insert(bucket.to_string(), vec![existing.clone()]);
for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] {
let base = sys
.targets_base_for_write(bucket, policy)
.await
.expect("a readable target set must be readable under either policy");
assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target");
assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target");
}
let base = sys.targets_base_for_write(bucket).await.expect("read targets");
assert_eq!(base.targets.len(), 1);
assert_eq!(base.targets[0].arn, existing.arn);
}
}
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
#[cfg(test)]
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
obj_name: &str,
rv_id: &str,
@@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
#[cfg(test)]
mod test {
#[cfg(feature = "test-util")]
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
use super::{
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use rustfs_filemeta::TransitionVersionState;
@@ -747,7 +747,7 @@ pub enum TransitionTransactionRecoveryOutcome {
OperatorRequired(IlmRecoveryErrorCode),
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
#[derive(Default)]
struct TransitionRecoveryClaimBarrierState {
transaction_id: Uuid,
@@ -755,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionRecoveryClaimBarrier {
state: Arc<TransitionRecoveryClaimBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl TransitionRecoveryClaimBarrier {
pub(crate) fn install(transaction_id: Uuid) -> Self {
let state = Arc::new(TransitionRecoveryClaimBarrierState {
@@ -796,7 +796,7 @@ impl TransitionRecoveryClaimBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionRecoveryClaimBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -810,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -825,7 +825,7 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
#[derive(Default)]
struct TransitionRecoveryTerminalBarrierState {
transaction_id: Uuid,
@@ -833,17 +833,17 @@ struct TransitionRecoveryTerminalBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionRecoveryTerminalBarrier {
state: Arc<TransitionRecoveryTerminalBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl TransitionRecoveryTerminalBarrier {
pub(crate) fn install(transaction_id: Uuid) -> Self {
let state = Arc::new(TransitionRecoveryTerminalBarrierState {
@@ -870,7 +870,7 @@ impl TransitionRecoveryTerminalBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionRecoveryTerminalBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -884,7 +884,7 @@ impl Drop for TransitionRecoveryTerminalBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -1242,7 +1242,7 @@ async fn process_transition_transaction_record_at(
},
)
.map_err(transition_transaction_store_error)?;
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_before_transition_recovery_claim(current.transaction_id).await;
match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
@@ -1300,7 +1300,7 @@ async fn process_transition_transaction_record_at(
};
persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
if let Some(source) = source_to_delete {
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_after_transition_recovery_terminal(source.transaction_id).await;
delete_transition_transaction_record(api, &source).await?;
}
@@ -1322,7 +1322,7 @@ fn transition_recovery_control_identity(transaction: &TransitionTransaction, rec
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
transition_recovery_control_identity(transaction, &record_name)
@@ -1753,7 +1753,7 @@ pub async fn recover_transition_transaction_records(
recover_transition_transaction_records_with_now(api, limit, marker, None).await
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pub async fn recover_transition_transaction_records_at(
api: Arc<ECStore>,
limit: usize,
+6
View File
@@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
#[cfg(unix)]
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
+1
View File
@@ -1355,6 +1355,7 @@ impl LocalDiskWrapper {
self.disk.get_object_path(volume, path)
}
#[cfg(unix)]
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
self.disk.get_object_path_for_io(volume, path)
}
+2
View File
@@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
fs::rename(from, to).await
}
#[cfg(any(not(windows), test))]
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
std::fs::rename(from, to)
}
#[cfg(any(not(windows), test))]
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
+21 -2
View File
@@ -735,7 +735,9 @@ const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_fa
/// to replace. Best effort — the rename that follows fails closed — but a
/// recurring signal means heal is stuck on that drive.
const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed";
#[cfg(unix)]
const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total";
#[cfg(unix)]
const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total";
// io_uring read-backend gray-release observability (rustfs/backlog#1172).
#[cfg(target_os = "linux")]
@@ -932,10 +934,15 @@ const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
#[cfg(any(unix, test))]
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
#[cfg(any(unix, test))]
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
#[cfg(any(unix, test))]
const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD";
#[cfg(any(unix, test))]
const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy";
#[cfg(any(unix, test))]
const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy";
/// Legacy binary switch for commit-point durability (fsync writes and renames).
@@ -951,6 +958,7 @@ const DEFAULT_RUSTFS_DRIVE_SYNC_ENABLE: bool = true;
/// See docs/operations/durability-modes.md for the power-loss guarantee matrix.
const ENV_RUSTFS_DURABILITY_MODE: &str = "RUSTFS_DURABILITY_MODE";
#[cfg(any(unix, test))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum LocalReadCopyMethod {
MmapCopy,
@@ -1359,15 +1367,18 @@ cached_read_env! {
cached_read_env! {
/// Whether mmap reads should fault the mapping in with `MAP_POPULATE`.
#[cfg(any(unix, test))]
fn mmap_populate_enabled() -> bool =
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE);
}
#[cfg(any(unix, test))]
fn should_populate_mmap_read(length: usize) -> bool {
length > 0 && mmap_populate_enabled()
}
cached_read_env! {
#[cfg(any(unix, test))]
fn local_read_copy_method() -> LocalReadCopyMethod = {
let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY);
match method.as_str() {
@@ -2010,7 +2021,7 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se
.insert(dst_path.to_string(), Box::new(hook));
}
#[cfg(test)]
#[cfg(all(test, unix))]
fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
INLINE_BEFORE_FILE_SYNC_ADMISSION
.lock()
@@ -2257,7 +2268,7 @@ fn should_remove_staged_meta_before_commit(_dst_path: &str) -> bool {
false
}
#[cfg(not(test))]
#[cfg(all(not(test), not(windows)))]
fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
false
}
@@ -3758,6 +3769,7 @@ struct FdKey {
/// The generation fence and explicit mutation invalidation keep the snapshot
/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
/// metadata syscall without weakening replacement/heal semantics.
#[cfg(unix)]
struct FdCacheEntry {
/// An independently cloneable descriptor for the immutable shard inode.
file: Arc<std::fs::File>,
@@ -5508,6 +5520,7 @@ impl LocalDisk {
local_disk_bucket_path(&self.root, bucket)
}
#[cfg(any(unix, test))]
pub(crate) fn get_object_path_for_io(&self, bucket: &str, key: &str) -> Result<PathBuf> {
self.io_get_object_path(bucket, key)
}
@@ -19351,11 +19364,17 @@ mod test {
path_resolve_stage: "path",
metadata_lookup_stage: "metadata_lookup",
metadata_validate_stage: "metadata_validate",
#[cfg(unix)]
blocking_wait_stage: "blocking_wait",
#[cfg(unix)]
blocking_task_stage: "blocking_task",
#[cfg(unix)]
file_open_stage: "file_open",
#[cfg(unix)]
mmap_map_stage: "mmap_map",
#[cfg(unix)]
mmap_copy_stage: "mmap_copy",
#[cfg(unix)]
direct_read_copy_stage: "direct_read_copy",
};
+4 -2
View File
@@ -17,13 +17,14 @@
#[cfg(all(test, windows))]
use super::run_destination_commit_directory_preparation;
#[cfg(any(not(windows), test))]
use super::should_fail_local_inline_rollback_hardlink;
use super::{
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
skip_access_checks,
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
};
#[cfg(test)]
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
@@ -88,6 +89,7 @@ fn rollback_inline_metadata_commit_std(
Ok(())
}
#[cfg(any(not(windows), test))]
pub(super) fn create_local_inline_rollback_backup(
dst_file_path: &Path,
staging_file_path: &Path,
+7
View File
@@ -198,11 +198,17 @@ pub struct MmapCopyStageMetrics {
pub(crate) path_resolve_stage: &'static str,
pub(crate) metadata_lookup_stage: &'static str,
pub(crate) metadata_validate_stage: &'static str,
#[cfg(unix)]
pub(crate) blocking_wait_stage: &'static str,
#[cfg(unix)]
pub(crate) blocking_task_stage: &'static str,
#[cfg(unix)]
pub(crate) file_open_stage: &'static str,
#[cfg(unix)]
pub(crate) mmap_map_stage: &'static str,
#[cfg(unix)]
pub(crate) mmap_copy_stage: &'static str,
#[cfg(unix)]
pub(crate) direct_read_copy_stage: &'static str,
}
@@ -949,6 +955,7 @@ impl Disk {
}
}
#[cfg(unix)]
pub(crate) fn get_object_path_for_io_if_local(
&self,
volume: &str,
+4
View File
@@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder {
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
#[cfg(unix)]
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
@@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder {
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
}
#[cfg(unix)]
pub(crate) fn record_limited(dir: &Path) {
record_path(&LIMITED, dir, "limited fsync dir recorder");
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
@@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder {
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
}
#[cfg(unix)]
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
BEFORE_LIMITED
.lock()
@@ -237,6 +240,7 @@ pub(crate) mod fsync_dir_recorder {
.insert(dir.to_path_buf(), kind);
}
#[cfg(unix)]
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
}
+12 -3
View File
@@ -13,12 +13,15 @@
// limitations under the License.
use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
};
#[cfg(unix)]
use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
};
#[cfg(feature = "hotpath")]
use crate::disk::FileWriter;
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
@@ -406,11 +409,17 @@ async fn open_disk_reader(
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
#[cfg(unix)]
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
#[cfg(unix)]
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
#[cfg(unix)]
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
#[cfg(unix)]
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
#[cfg(unix)]
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
#[cfg(unix)]
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
});
let mmap_result = {
@@ -56,6 +56,7 @@
use std::collections::HashMap;
use std::io::Cursor;
#[cfg(feature = "test-util")]
use std::path::Path;
use std::sync::{
Arc,
@@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt;
use tokio::sync::{Mutex, Notify, RwLock};
use uuid::Uuid;
#[cfg(feature = "test-util")]
use crate::disk::endpoint::Endpoint;
#[cfg(feature = "test-util")]
use crate::disk::format::FormatV3;
#[cfg(feature = "test-util")]
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
use crate::services::tier::tier::TierConfigMgr;
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
use crate::services::tier::warm_backend::{
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
#[cfg(feature = "test-util")]
use rustfs_filemeta::FileMeta;
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
#[cfg(feature = "test-util")]
use rustfs_utils::path::path_join_buf;
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
#[cfg(feature = "test-util")]
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
#[cfg(feature = "test-util")]
impl TransitionCleanupStoreBarrier {
/// Install the barrier for the next rejected transition cleanup.
pub fn install() -> Self {
@@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier {
}
/// Default polling cadence used by the `wait_for_*` helpers.
#[cfg(feature = "test-util")]
const POLL_INTERVAL: Duration = Duration::from_millis(50);
/// A fault to inject into [`MockWarmBackend`] operations.
@@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard {
}
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
#[cfg(feature = "test-util")]
pub struct MockPutBarrier {
state: Arc<MockPutBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockPutBarrier {
/// Wait until the remote body is stored and the PUT is paused before returning.
pub async fn wait_until_paused(&self) {
@@ -226,6 +237,7 @@ impl MockPutBarrier {
}
}
#[cfg(feature = "test-util")]
impl Drop for MockPutBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -258,10 +270,12 @@ impl Drop for MockGetBarrier {
}
/// One-shot barrier that pauses and then fails a mock tier DELETE.
#[cfg(feature = "test-util")]
pub struct MockRemoveBarrier {
state: Arc<MockRemoveBarrierState>,
}
#[cfg(feature = "test-util")]
impl MockRemoveBarrier {
/// Wait until DELETE reaches the deterministic failure point.
pub async fn wait_until_paused(&self) {
@@ -283,6 +297,7 @@ impl MockRemoveBarrier {
}
}
#[cfg(feature = "test-util")]
impl Drop for MockRemoveBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -306,6 +321,7 @@ impl MockWarmBackend {
}
/// Arm a one-shot pause after the next tier PUT stores its remote body.
#[cfg(feature = "test-util")]
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
let state = Arc::new(MockPutBarrierState::default());
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
@@ -313,6 +329,7 @@ impl MockWarmBackend {
}
/// Pause and then fail the next DELETE after it reaches the backend.
#[cfg(feature = "test-util")]
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
let state = Arc::new(MockRemoveBarrierState::default());
let mut barrier = self.inner.remove_barrier.lock().await;
@@ -323,6 +340,7 @@ impl MockWarmBackend {
/// Arm a one-shot pause before the next tier GET, then return an error
/// after the test releases it.
#[cfg(feature = "test-util")]
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
let state = Arc::new(MockGetBarrierState {
fail_after_release: true,
@@ -343,6 +361,7 @@ impl MockWarmBackend {
// ---- fault injection -------------------------------------------------
/// Replace the entire fault configuration.
#[cfg(feature = "test-util")]
pub async fn set_faults(&self, faults: FaultConfig) {
*self.inner.faults.lock().await = faults;
}
@@ -353,6 +372,7 @@ impl MockWarmBackend {
}
/// Toggle "HTTP 5xx" server errors on every operation.
#[cfg(feature = "test-util")]
pub async fn set_server_error(&self, server_error: bool) {
self.inner.faults.lock().await.server_error = server_error;
}
@@ -363,11 +383,13 @@ impl MockWarmBackend {
}
/// Set (or clear, with `None`) injected latency applied before each op.
#[cfg(feature = "test-util")]
pub async fn set_latency(&self, latency: Option<Duration>) {
self.inner.faults.lock().await.latency = latency;
}
/// Clear all injected faults, restoring healthy behaviour.
#[cfg(feature = "test-util")]
pub async fn clear_faults(&self) {
*self.inner.faults.lock().await = FaultConfig::default();
}
@@ -375,6 +397,7 @@ impl MockWarmBackend {
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
/// the complete body. This models a backend that incorrectly accepts a
/// truncated stream while still returning success.
#[cfg(feature = "test-util")]
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
*self.inner.put_read_limit.lock().await = limit;
}
@@ -395,12 +418,14 @@ impl MockWarmBackend {
}
/// Reject non-empty remote versions before transition metadata is committed.
#[cfg(feature = "test-util")]
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
}
/// Reject the next non-empty remote version validation without changing
/// subsequent exact-version backend cleanup behavior.
#[cfg(feature = "test-util")]
pub fn reject_next_non_empty_remote_version_validation(&self) {
self.inner
.reject_non_empty_remote_version_validations
@@ -438,6 +463,7 @@ impl MockWarmBackend {
}
/// Clear the operation log without touching stored objects or faults.
#[cfg(feature = "test-util")]
pub async fn clear_op_log(&self) {
self.inner.op_log.lock().await.clear();
}
@@ -459,11 +485,13 @@ impl MockWarmBackend {
}
/// Return the exact object/version pairs produced by successful tier PUTs.
#[cfg(feature = "test-util")]
pub async fn put_versions(&self) -> Vec<(String, String)> {
self.inner.put_versions.lock().await.clone()
}
/// Return the exact object/version pairs passed to successful tier removes.
#[cfg(feature = "test-util")]
pub async fn remove_versions(&self) -> Vec<(String, String)> {
self.inner.remove_versions.lock().await.clone()
}
@@ -475,6 +503,7 @@ impl MockWarmBackend {
/// Number of `get` calls recorded — useful to assert restore reads hit the
/// local copy rather than the remote tier.
#[cfg(feature = "test-util")]
pub async fn get_count(&self) -> usize {
self.inner
.op_log
@@ -486,6 +515,7 @@ impl MockWarmBackend {
}
/// Number of `put` calls recorded.
#[cfg(feature = "test-util")]
pub async fn put_count(&self) -> usize {
self.inner
.op_log
@@ -499,6 +529,7 @@ impl MockWarmBackend {
// ---- storage inspection ---------------------------------------------
/// Whether the backend currently stores `object`.
#[cfg(feature = "test-util")]
pub async fn contains(&self, object: &str) -> bool {
self.inner.objects.lock().await.contains_key(object)
}
@@ -509,11 +540,13 @@ impl MockWarmBackend {
}
/// A clone of the stored object, if present.
#[cfg(feature = "test-util")]
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
self.inner.objects.lock().await.get(object).cloned()
}
/// A clone of the raw bytes stored for `object`, if present.
#[cfg(feature = "test-util")]
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
}
@@ -538,6 +571,7 @@ impl MockWarmBackend {
/// Poll until `object` is absent from the backend, or `timeout` elapses.
/// Returns `true` if the object disappeared within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -553,6 +587,7 @@ impl MockWarmBackend {
/// Poll until the backend holds exactly `expected` objects, or `timeout`
/// elapses. Returns `true` if the count was reached within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
/// free-version count.
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg(feature = "test-util")]
pub struct TransitionMeta {
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
pub status: String,
@@ -860,6 +896,7 @@ pub struct TransitionMeta {
pub free_version_count: usize,
}
#[cfg(feature = "test-util")]
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
// with the position recorded in the disk's own format.json, so derive the
@@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
/// The free-version metadata removal lands asynchronously after the remote
/// object disappears, so callers typically poll via
/// [`wait_for_free_version_absence`] instead of asserting a single read.
#[cfg(feature = "test-util")]
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
let Some(disk) = open_disk(disk_path).await else {
return 0;
@@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
/// fields are taken from the newest version that carries a transition record;
/// if no version is transitioned, they are taken from the current version (and
/// will be empty).
#[cfg(feature = "test-util")]
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
let disk = open_disk(disk_path).await?;
let data = disk
@@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
/// disk is missing the object or disagrees — this is the shard-consistency
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
/// four-tuple plus free-version count must match across all erasure shards).
#[cfg(feature = "test-util")]
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
@@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
/// elapses. Returns `true` if the free versions drained within the budget.
#[cfg(feature = "test-util")]
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
@@ -1040,6 +1081,44 @@ mod tests {
);
}
#[tokio::test]
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
let backend = MockWarmBackend::new();
let metadata = HashMap::from([
("content-type".to_string(), "text/plain".to_string()),
("project".to_string(), "archive".to_string()),
]);
let version = backend
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
.await
.expect("mock PUT should preserve remote metadata");
assert_eq!(backend.metadata("object").await, Some(metadata));
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe stored object"),
TransitionCandidateProbe::VersionedPresent(version)
);
backend.external_remove("object").await;
assert_eq!(backend.metadata("object").await, None);
assert_eq!(
backend
.probe_transition_candidate_state("object")
.await
.expect("probe removed object"),
TransitionCandidateProbe::Missing
);
let operations = backend.op_log().await;
assert!(
operations
.iter()
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
);
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
}
#[tokio::test]
async fn mock_probe_preserves_fault_fail_closed_behavior() {
let backend = MockWarmBackend::new();
@@ -111,10 +111,11 @@ use crate::disk::{
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
use crate::io_support::bitrot::{
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
object_mmap_read_max_length,
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
create_deferred_bitrot_reader_with_stripe_handle,
};
#[cfg(unix)]
use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
use crate::set_disk::runtime_sources;
use crate::set_disk::shard_source::ShardReadCost;
use crate::storage_api_contracts::object::ObjectOperations;
+1 -1
View File
@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
+73 -14
View File
@@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
}
async fn transition_cleanup_store(ctx: &Arc<crate::runtime::instance::InstanceContext>) -> Option<Arc<ECStore>> {
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
pause_transition_cleanup_store().await;
transition_object_store(ctx).await
@@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup(
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
#[derive(Default)]
struct TransitionCleanupStoreBarrierState {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
/// One-shot test barrier placed before transition cleanup resolves its ECStore.
pub(crate) struct TransitionCleanupStoreBarrier {
state: Arc<TransitionCleanupStoreBarrierState>,
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCleanupStoreBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
impl TransitionCleanupStoreBarrier {
/// Install the process-local barrier for the next cleanup-store resolution.
pub(crate) fn install() -> Self {
@@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier {
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
impl Drop for TransitionCleanupStoreBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier {
}
}
#[cfg(any(test, feature = "test-util"))]
#[cfg(feature = "test-util")]
async fn pause_transition_cleanup_store() {
let barrier = TRANSITION_CLEANUP_STORE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
struct TransitionUploadedCommitBarrierState {
bucket: String,
object: String,
@@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState {
release: tokio::sync::Notify,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) struct TransitionUploadedCommitBarrier {
state: Arc<TransitionUploadedCommitBarrierState>,
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<TransitionUploadedCommitBarrierState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl TransitionUploadedCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(TransitionUploadedCommitBarrierState {
@@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
impl Drop for TransitionUploadedCommitBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
@@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier {
}
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) {
let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
upload_cleanup.update_cleanup_transaction(&transaction);
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pause_after_transition_uploaded_persisted(bucket, object).await;
let commit_opts = opts.as_commit_opts();
@@ -12672,6 +12672,65 @@ mod metadata_mutation_generation_tests {
set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
}
#[tokio::test]
#[serial_test::serial(metadata_cache_invalidation_probe)]
async fn segment_observation_equal_size_mutations_retire_metadata_generation() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "segment-observation-bucket";
let object = "hot/object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("create segment fixture bucket");
}
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await;
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
let mut replacement = PutObjReader::from_vec(b"after!".to_vec());
set_disks
.put_object(bucket, object, &mut replacement, &ObjectOptions::default())
.await
.expect("commit same-length replacement with normal owner locking");
assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation");
assert_retired(&set_disks, &old_key).await;
drop(probe);
let after = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("read replacement metadata");
assert_eq!(before.size, after.size);
assert_ne!(before.etag, after.etag, "equal size is not equal content");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read replacement body through the owner");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("drain replacement body");
assert_eq!(body, b"after!");
drop(reader);
let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await;
let probe = MetadataCacheInvalidationProbe::install(bucket, object);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])),
..Default::default()
},
)
.await
.expect("commit metadata-only mutation with normal owner locking");
assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences");
assert_retired(&set_disks, &old_key).await;
let after = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("read committed metadata-only mutation");
assert_eq!(before.size, after.size);
assert_eq!(before.etag, after.etag);
assert!(!before.user_defined.contains_key("x-amz-meta-segment"));
assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed"));
}
#[tokio::test]
#[serial_test::serial(metadata_cache_invalidation_probe)]
async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
+2 -2
View File
@@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation(
}
impl ECStore {
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
data_movement_multipart_discovery_counts()
.lock()
@@ -246,7 +246,7 @@ impl ECStore {
.insert(self.id, 0);
}
#[cfg(test)]
#[cfg(all(test, feature = "test-util"))]
pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
data_movement_multipart_discovery_counts()
.lock()
+9
View File
@@ -54,6 +54,15 @@ pub enum Error {
#[error("Heal task execution failed: {message}")]
TaskExecutionFailed { message: String },
/// The current page already exhausted its local retry budget. Retrying
/// the enclosing bucket would replay pages whose results were counted.
#[error("Heal listing failed for bucket {bucket}: {source}")]
HealListingFailed {
bucket: String,
#[source]
source: Box<Error>,
},
#[error("Invalid heal type: {heal_type}")]
InvalidHealType { heal_type: String },
+6
View File
@@ -447,6 +447,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"running".to_string(),
None,
@@ -463,6 +464,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"running".to_string(),
Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
@@ -479,6 +481,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"finished".to_string(),
None,
@@ -495,6 +498,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"stopped".to_string(),
Some("heal task cancelled".to_string()),
@@ -511,6 +515,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"stopped".to_string(),
Some("heal task timed out".to_string()),
@@ -527,6 +532,7 @@ impl HealChannelProcessor {
progress,
next_seq,
min_seq,
..
}) => (
"stopped".to_string(),
Some(error),
+6
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::heal::{
outcome::HealTaskOutcome,
progress::{HealProgress, HealStatistics},
resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
storage::HealStorageAPI,
@@ -185,6 +186,7 @@ fn record_displaced_terminal(
request: &HealRequest,
) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus {
outcome: None,
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
@@ -268,6 +270,7 @@ async fn publish_completed_heal(
#[derive(Debug, Clone)]
pub struct HealTaskReport {
pub outcome: Option<Arc<HealTaskOutcome>>,
pub status: HealTaskStatus,
pub result_items: Vec<HealResultItem>,
pub result_items_truncated: bool,
@@ -285,6 +288,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
let window = task.get_result_items_since(since).await;
HealTaskReport {
status: task.get_status().await,
outcome: Some(Arc::new(task.get_outcome().await)),
result_items: window.items,
// The legacy flag stays set once anything was evicted; a lagging
// incremental cursor additionally marks this response truncated so
@@ -298,6 +302,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
HealTaskReport {
outcome: None,
status,
result_items: Vec::new(),
result_items_truncated: false,
@@ -325,6 +330,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
};
HealTaskReport {
status: completed.status.clone(),
outcome: completed.outcome.clone(),
result_items,
result_items_truncated: completed.result_items_truncated || lagged,
progress: completed.progress.clone(),
+3
View File
@@ -83,6 +83,7 @@ pub(super) struct CompletedHealStatus {
pub(super) heal_type: HealType,
pub(super) status: HealTaskStatus,
pub(super) progress: Option<HealProgress>,
pub(super) outcome: Option<Arc<HealTaskOutcome>>,
pub(super) retained_bytes: std::sync::OnceLock<usize>,
pub(super) result_items_truncated: bool,
pub(super) completed_at: SystemTime,
@@ -105,6 +106,7 @@ impl CompletedHealStatus {
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
match &self.heal_type {
HealType::Cluster => {}
HealType::Bucket { bucket } => add(bucket.capacity()),
@@ -209,6 +211,7 @@ impl CompletedHealStatus {
heal_type: task.heal_type.clone(),
status,
progress: Some(task.get_progress().await),
outcome: Some(Arc::new(task.get_outcome().await)),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: task.result_items_truncated(),
completed_at: SystemTime::now(),
@@ -298,6 +298,7 @@ impl HealManager {
if cancelled_completion {
completed_status = HealTaskStatus::Cancelled;
completed_status_entry.status = HealTaskStatus::Cancelled;
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
}
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
+64
View File
@@ -103,6 +103,7 @@ struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
outcome: None,
heal_type: HealType::Cluster,
status: HealTaskStatus::Completed,
progress: Some(HealProgress {
@@ -287,6 +288,59 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat
}
}
#[tokio::test]
async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
let bucket = "canonical-outcome-cancel-before-finish";
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let alias = duplicate.id.clone();
let retention_hook = Arc::new(CompletedRetentionHook::default());
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.to_string(), retention_hook.clone());
hooks.insert(task_id.clone(), retention_hook.clone());
}
let finish_hook = Arc::new(OutcomeFinishTestHook {
task_id: task_id.clone(),
reached: Notify::new(),
release: Notify::new(),
});
*OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone());
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified())
.await
.expect("storage started");
retention_hook.execute.notify_one();
tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified())
.await
.expect("storage returned before outcome finalization");
manager.cancel_task(&alias).await.expect("cancel wins publication");
finish_hook.release.notify_one();
tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified())
.await
.expect("scheduler completes cancelled handoff");
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(
report.outcome.as_ref().expect("frozen outcome").execution,
HealExecutionOutcome::Aborted(HealAbortReason::Cancelled)
);
}
retention_hook.finish.notify_one();
*OUTCOME_FINISH_TEST_HOOK.lock().await = None;
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != bucket && key != &task_id);
}
#[tokio::test]
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
let bucket = "completed-retention-retry-cancel";
@@ -325,6 +379,10 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(
report.outcome.as_ref().expect("cancelled outcome retained").execution,
crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled)
);
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
}
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
@@ -391,6 +449,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
.expect("scheduler archives terminal");
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
let expected = task.get_progress().await;
let expected_outcome = task.get_outcome().await;
for token in [&task_id, &alias] {
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
let report = manager
@@ -398,6 +457,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
.await
.expect("terminal token remains queryable at handoff");
assert_eq!(report.progress.as_ref(), Some(&expected));
assert_eq!(report.outcome.as_deref(), Some(&expected_outcome));
assert!(report.result_items.is_empty());
match outcome {
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
@@ -1976,6 +2036,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
task_id,
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type,
status: HealTaskStatus::Retrying {
@@ -2700,6 +2761,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
task_id.clone(),
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Retrying {
@@ -2737,6 +2799,7 @@ async fn test_get_task_status_reads_recent_completed_status() {
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket {
bucket: "bucket".to_string(),
@@ -2768,6 +2831,7 @@ async fn test_get_task_report_for_path_reads_completed_items() {
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object {
bucket: "bucket".to_string(),
+1
View File
@@ -16,6 +16,7 @@ pub mod channel;
pub mod erasure_healer;
pub mod manager;
pub mod mrf_queue;
pub mod outcome;
pub mod progress;
pub(crate) mod replacement_readiness;
pub mod resume;
@@ -33,6 +33,9 @@ use std::collections::HashMap;
use tokio::io::AsyncReadExt;
use uuid::Uuid;
/// Explicit pending migration; never activates the production writer or GC.
pub mod migration;
// Root-level control files avoid requiring a new directory before the first
// atomic commit. They remain inside the storage owner's metadata volume.
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
@@ -66,6 +69,23 @@ struct Manifest {
}
impl Manifest {
fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
bytes.extend_from_slice(MAGIC);
bytes.push(VERSION);
bytes.extend_from_slice(owner.as_bytes());
bytes.extend_from_slice(&sequence.to_le_bytes());
bytes.extend_from_slice(
&u64::try_from(payload.len())
.map_err(|_| SnapshotError::TooLarge)?
.to_le_bytes(),
);
bytes.extend_from_slice(&Sha256::digest(payload));
bytes.extend_from_slice(&Sha256::digest(&bytes));
Self::decode(&bytes, payload.len())?;
Ok(bytes)
}
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
return Err(SnapshotError::Corrupt);
File diff suppressed because it is too large Load Diff
+305
View File
@@ -0,0 +1,305 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Execution results are separate from repair responsibility. A legacy
//! successful storage call supplies no authoritative repair receipt.
use std::{collections::VecDeque, time::SystemTime};
use uuid::Uuid;
const MAX_OUTCOME_ITEMS: usize = 128;
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealObjectKind {
Object,
Metadata,
Decode,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HealObjectIdentity {
pub kind: HealObjectKind,
pub bucket: String,
pub object: String,
/// The requested version; None remains unresolved, never an absence proof.
pub version_id: Option<String>,
pub bucket_incarnation_id: Option<Uuid>,
pub pool_index: Option<usize>,
pub set_index: Option<usize>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealDeferredReason {
DanglingDeleteGrace,
TransientUsageCache,
TransientExistenceCheck,
Deadline,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealFailureClass {
Recoverable,
RetryExhausted,
Permanent,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HealObjectDisposition {
/// The legacy storage response does not prove the requested check or commit.
Unknown,
Repaired,
VerifiedHealthy,
AuthoritativelyAbsent,
Deferred {
reason: HealDeferredReason,
retry_not_before: Option<SystemTime>,
},
Failed(HealFailureClass),
Cancelled,
DryRunObserved,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HealObjectOutcome {
pub identity: HealObjectIdentity,
pub disposition: HealObjectDisposition,
pub detail: Option<String>,
}
impl HealObjectOutcome {
fn retained_bytes(&self) -> usize {
size_of::<Self>()
.saturating_add(self.identity.bucket.capacity())
.saturating_add(self.identity.object.capacity())
.saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity))
.saturating_add(self.detail.as_ref().map_or(0, String::capacity))
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum HealTraversalCoverage {
#[default]
Unknown,
Partial,
Complete,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealAbortReason {
Cancelled,
Deadline,
Untraversable,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum HealExecutionOutcome {
#[default]
Pending,
Running,
Completed,
CompletedWithErrors,
Aborted(HealAbortReason),
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct HealOutcomeCounters {
pub processed: u64,
pub healed: u64,
pub unchanged: u64,
/// Deferred, cancelled, dry-run and unverified results remain unresolved.
pub skipped: u64,
pub failed: u64,
pub unknown: u64,
pub attempt_failures: u64,
pub overflowed: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct HealTaskOutcome {
pub execution: HealExecutionOutcome,
pub coverage: HealTraversalCoverage,
pub counters: HealOutcomeCounters,
/// A bounded diagnostic window, not a complete responsibility ledger.
pub objects: VecDeque<HealObjectOutcome>,
pub objects_truncated: bool,
retained_object_bytes: usize,
untraversable: bool,
}
impl HealTaskOutcome {
pub(crate) fn start(&mut self) {
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
self.execution = HealExecutionOutcome::Running;
}
self.coverage = HealTraversalCoverage::Partial;
}
pub(crate) fn attempt_failed(&mut self) {
self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures);
}
pub(crate) fn mark_untraversable(&mut self) {
self.untraversable = true;
self.coverage = HealTraversalCoverage::Partial;
}
pub(crate) fn finish(&mut self, abort: Option<HealAbortReason>) {
if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
return;
}
let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable));
self.execution = match abort {
Some(reason) => HealExecutionOutcome::Aborted(reason),
None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors,
None => HealExecutionOutcome::Completed,
};
self.coverage = if abort.is_none() && !self.counters.overflowed {
HealTraversalCoverage::Complete
} else {
HealTraversalCoverage::Partial
};
}
pub(crate) fn record(&mut self, mut item: HealObjectOutcome) {
use super::progress::increment_counter;
let counters = &mut self.counters;
counters.overflowed |= !increment_counter(&mut counters.processed);
let counter = match item.disposition {
HealObjectDisposition::Repaired => &mut counters.healed,
HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
HealObjectDisposition::Failed(_) => &mut counters.failed,
HealObjectDisposition::Unknown => {
counters.overflowed |= !increment_counter(&mut counters.unknown);
&mut counters.skipped
}
_ => &mut counters.skipped,
};
counters.overflowed |= !increment_counter(counter);
if let Some(detail) = &mut item.detail {
let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES);
while !detail.is_char_boundary(end) {
end -= 1;
}
self.objects_truncated |= end < detail.len();
detail.truncate(end);
detail.shrink_to_fit();
}
let bytes = item.retained_bytes();
if bytes > MAX_OUTCOME_BYTES {
self.objects_truncated = true;
return;
}
while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES {
let Some(oldest) = self.objects.pop_front() else { break };
self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes());
self.objects_truncated = true;
}
self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes);
self.objects.push_back(item);
}
pub(crate) fn retained_bytes(&self) -> usize {
size_of::<Self>()
.saturating_add(self.retained_object_bytes)
.saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
}
}
#[cfg(test)]
mod canonical_outcome_tests {
use super::*;
fn item(disposition: HealObjectDisposition) -> HealObjectOutcome {
HealObjectOutcome {
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
bucket_incarnation_id: None,
pool_index: None,
set_index: None,
},
disposition,
detail: None,
}
}
#[test]
fn canonical_outcome_categories_have_one_terminal_count() {
let mut outcome = HealTaskOutcome::default();
for disposition in [
HealObjectDisposition::Unknown,
HealObjectDisposition::Repaired,
HealObjectDisposition::VerifiedHealthy,
HealObjectDisposition::AuthoritativelyAbsent,
HealObjectDisposition::Deferred {
reason: HealDeferredReason::DanglingDeleteGrace,
retry_not_before: None,
},
HealObjectDisposition::Failed(HealFailureClass::Permanent),
HealObjectDisposition::Cancelled,
HealObjectDisposition::DryRunObserved,
] {
outcome.record(item(disposition));
}
let c = &outcome.counters;
assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
}
#[test]
fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() {
let mut outcome = HealTaskOutcome::default();
for _ in 0..MAX_OUTCOME_ITEMS {
outcome.record(item(HealObjectDisposition::Unknown));
}
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
assert!(!outcome.objects_truncated);
outcome.record(item(HealObjectDisposition::Unknown));
assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
assert!(outcome.objects_truncated);
let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES);
outcome.record(oversized);
assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count"));
assert_eq!(outcome.counters.failed, 1);
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
for _ in 0..MAX_OUTCOME_ITEMS {
let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES));
outcome.record(failed);
}
assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
assert!(outcome.objects.iter().all(|item| {
item.detail
.as_ref()
.is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES)
}));
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
}
#[test]
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
let mut outcome = HealTaskOutcome::default();
outcome.counters.processed = u64::MAX;
outcome.record(item(HealObjectDisposition::Unknown));
outcome.finish(None);
assert!(outcome.counters.overflowed);
assert_eq!(outcome.counters.processed, u64::MAX);
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
}
}
+132
View File
@@ -15,6 +15,10 @@
use crate::heal::{
DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _,
erasure_healer::target_outcomes_complete,
outcome::{
HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind,
HealObjectOutcome, HealTaskOutcome,
},
progress::HealProgress,
resume::{
CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
@@ -43,6 +47,26 @@ use uuid::Uuid;
use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
#[cfg(test)]
pub(crate) struct OutcomeFinishTestHook {
pub(crate) task_id: String,
pub(crate) reached: tokio::sync::Notify,
pub(crate) release: tokio::sync::Notify,
}
#[cfg(test)]
pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
#[cfg(test)]
async fn pause_outcome_finish(task_id: &str) {
let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
hook.reached.notify_one();
hook.release.notified().await;
}
}
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_TASK: &str = "task";
const LOG_SUBSYSTEM_OBJECT: &str = "object";
@@ -394,6 +418,7 @@ pub struct HealTask {
pub status: Arc<RwLock<HealTaskStatus>>,
/// Progress tracking
pub progress: Arc<RwLock<HealProgress>>,
outcome: Arc<RwLock<HealTaskOutcome>>,
/// Result items collected from storage heal calls, each stamped with a
/// monotonically increasing sequence number for incremental consumption
/// (the client passes the last seen seq back and receives only newer
@@ -460,6 +485,7 @@ impl HealTask {
result_items_truncated: Arc::new(AtomicBool::new(false)),
batch_failure: Arc::new(RwLock::new(None)),
batch_failure_recorded: Arc::new(AtomicBool::new(false)),
outcome: Arc::new(RwLock::new(HealTaskOutcome::default())),
created_at: request.created_at,
enqueued_at: request.enqueued_at,
started_at: Arc::new(RwLock::new(None)),
@@ -507,6 +533,66 @@ impl HealTask {
self.heal_type.kind_label()
}
pub async fn get_outcome(&self) -> HealTaskOutcome {
self.outcome.read().await.clone()
}
fn outcome_identity(
&self,
bucket: &str,
object: &str,
version_id: Option<&str>,
pool_index: Option<usize>,
set_index: Option<usize>,
) -> HealObjectIdentity {
HealObjectIdentity {
kind: match self.heal_type {
HealType::Metadata { .. } => HealObjectKind::Metadata,
HealType::ECDecode { .. } => HealObjectKind::Decode,
_ => HealObjectKind::Object,
},
bucket: bucket.to_owned(),
object: object.to_owned(),
version_id: version_id.map(ToOwned::to_owned),
bucket_incarnation_id: None,
pool_index,
set_index,
}
}
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
let (bucket, object, version) = match &self.heal_type {
HealType::Object {
bucket,
object,
version_id,
}
| HealType::ECDecode {
bucket,
object,
version_id,
} => (bucket, object, version_id.as_deref()),
HealType::Metadata { bucket, object } => (bucket, object, None),
_ => return None,
};
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
}
async fn record_deferred_object(&self, reason: HealDeferredReason) {
if let Some(identity) = self.single_object_identity() {
let mut outcome = self.outcome.write().await;
outcome.attempt_failed();
outcome.record(HealObjectOutcome {
identity,
disposition: HealObjectDisposition::Deferred {
reason,
retry_not_before: None,
},
detail: None,
});
}
}
pub(crate) fn has_batch_failure(&self) -> bool {
self.batch_failure_recorded.load(Ordering::Acquire)
}
@@ -634,6 +720,7 @@ impl HealTask {
}
async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await;
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
@@ -733,6 +820,8 @@ impl HealTask {
return false;
}
self.record_deferred_object(HealDeferredReason::TransientUsageCache).await;
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
@@ -755,6 +844,8 @@ impl HealTask {
return false;
}
self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await;
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_OBJECT_RESULT,
@@ -801,6 +892,7 @@ impl HealTask {
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
#[hotpath::measure]
pub async fn execute(&self) -> Result<()> {
self.outcome.write().await.start();
// update status and timestamps atomically to avoid race conditions
let now = SystemTime::now();
let start_instant = Instant::now();
@@ -860,6 +952,45 @@ impl HealTask {
HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
};
#[cfg(test)]
pause_outcome_finish(&self.id).await;
{
let mut outcome = self.outcome.write().await;
if outcome.counters.processed == 0
&& let Some(identity) = self.single_object_identity()
{
let disposition = match &result {
Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved,
Ok(()) => HealObjectDisposition::Unknown,
Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled,
Err(Error::TaskTimeout) => HealObjectDisposition::Deferred {
reason: HealDeferredReason::Deadline,
retry_not_before: None,
},
Err(error) => {
outcome.attempt_failed();
HealObjectDisposition::Failed(if error.is_recoverable_heal() {
HealFailureClass::Recoverable
} else {
HealFailureClass::Permanent
})
}
};
outcome.record(HealObjectOutcome {
identity,
disposition,
detail: result.as_ref().err().map(ToString::to_string),
});
}
let abort = match &result {
Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled),
Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline),
Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable),
_ => None,
};
outcome.finish(abort);
}
// update completed time and status
{
let mut completed_at = self.completed_at.write().await;
@@ -944,6 +1075,7 @@ impl HealTask {
pub async fn cancel(&self) -> Result<()> {
self.cancel_token.cancel();
self.outcome.write().await.finish(Some(HealAbortReason::Cancelled));
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
debug!(
+98 -19
View File
@@ -214,6 +214,7 @@ impl HealTask {
continue;
}
failed = failed.saturating_add(1);
self.outcome.write().await.mark_untraversable();
if err.is_recoverable_heal() {
retryable = retryable.saturating_add(1);
} else {
@@ -260,6 +261,7 @@ impl HealTask {
#[hotpath::measure]
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
let previous_progress = self.get_progress().await;
let mut scanned = 0u64;
let mut healed = 0u64;
let mut failed = 0u64;
@@ -304,23 +306,47 @@ impl HealTask {
let mut continuation_token: Option<String> = None;
loop {
self.check_control_flags().await?;
let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() {
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
set_disk_id,
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await?
} else {
self.await_with_control(self.storage.list_objects_for_heal_page(
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await?
let mut listing_attempt = 0;
let (objects, next_token, is_truncated) = loop {
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
set_disk_id,
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await
} else {
self.await_with_control(self.storage.list_objects_for_heal_page(
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await
};
match page {
Ok(page) => break page,
Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
Err(error) => {
self.outcome.write().await.attempt_failed();
if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
listing_attempt += 1;
self.await_with_control(async {
tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await;
Ok(())
})
.await?;
continue;
}
self.outcome.write().await.mark_untraversable();
return Err(Error::HealListingFailed {
bucket: bucket.to_string(),
source: Box::new(error),
});
}
}
};
let mut pending = objects;
@@ -338,6 +364,14 @@ impl HealTask {
self.check_control_flags().await?;
let mut telemetry_unknown = false;
let object = item.name.as_str();
let identity =
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
let mut disposition = if heal_opts.dry_run {
HealObjectDisposition::DryRunObserved
} else {
HealObjectDisposition::Unknown
};
let mut detail = None;
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
@@ -380,7 +414,31 @@ impl HealTask {
};
if let Some(err) = error {
match err {
Error::TaskCancelled | Error::TaskTimeout => {
let disposition = if matches!(err, Error::TaskCancelled) {
HealObjectDisposition::Cancelled
} else {
HealObjectDisposition::Deferred {
reason: HealDeferredReason::Deadline,
retry_not_before: None,
}
};
self.outcome.write().await.record(HealObjectOutcome {
identity,
disposition,
detail: None,
});
return Err(err);
}
_ => self.outcome.write().await.attempt_failed(),
}
detail = Some(err.to_string());
if Self::is_dangling_delete_grace_error(&err) {
disposition = HealObjectDisposition::Deferred {
reason: HealDeferredReason::DanglingDeleteGrace,
retry_not_before: None,
};
telemetry_unknown |= !increment_counter(&mut skipped);
warn!(
target: "rustfs::heal::task",
@@ -395,6 +453,10 @@ impl HealTask {
"Heal bucket object dangling cleanup deferred by grace window"
);
} else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
disposition = HealObjectDisposition::Deferred {
reason: HealDeferredReason::TransientUsageCache,
retry_not_before: None,
};
telemetry_unknown |= !increment_counter(&mut skipped);
warn!(
target: "rustfs::heal::task",
@@ -425,6 +487,11 @@ impl HealTask {
);
retry.push(item);
} else {
disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() {
HealFailureClass::RetryExhausted
} else {
HealFailureClass::Permanent
});
telemetry_unknown |= !increment_counter(&mut failed);
if err.is_recoverable_heal() {
retryable_failed = retryable_failed.saturating_add(1);
@@ -459,8 +526,20 @@ impl HealTask {
continue;
}
self.outcome.write().await.record(HealObjectOutcome {
identity,
disposition,
detail,
});
let mut progress = self.progress.write().await;
progress.update_object_progress(scanned, healed, failed, skipped, bytes);
progress.update_object_progress(
previous_progress.objects_scanned.saturating_add(scanned),
previous_progress.objects_healed.saturating_add(healed),
previous_progress.objects_failed.saturating_add(failed),
previous_progress.skipped_objects.saturating_add(skipped),
previous_progress.bytes_processed.saturating_add(bytes),
);
if telemetry_unknown {
progress.mark_unknown();
}
@@ -475,7 +554,7 @@ impl HealTask {
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
if continuation_token.is_none() {
// Truncated but no continuation token: end of listing.
// Truncated without a continuation token is a compatibility EOF.
break;
}
}
+2 -2
View File
@@ -261,8 +261,8 @@ impl HealTask {
update_parity: true,
no_lock: self.options.no_lock,
read_repair: false,
pool: None,
set: None,
pool: self.options.pool_index,
set: self.options.set_index,
};
let heal_result = self
+401 -2
View File
@@ -14,6 +14,364 @@
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
use super::*;
mod canonical_outcome {
use super::*;
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
HealTask::from_request(
HealRequest::new(
HealType::Bucket {
bucket: "bucket-a".to_string(),
},
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage,
)
}
#[tokio::test(start_paused = true)]
async fn cluster_retries_only_the_failed_listing_page() {
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(1)),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage.clone(),
);
task.execute().await.expect("second-page retry succeeds");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!(outcome.counters.processed, 2);
assert_eq!(outcome.counters.attempt_failures, 1);
assert_eq!(task.get_progress().await.objects_scanned, 2);
assert_eq!(
storage.heal_object_calls.lock().expect("object calls").as_slice(),
["object-a", "object-b"]
);
assert_eq!(
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
[None, Some("second".to_string()), Some("second".to_string())]
);
}
#[tokio::test(start_paused = true)]
async fn exhausted_listing_page_cannot_restart_the_bucket() {
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(4)),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage.clone(),
);
task.execute().await.expect_err("listing page budget exhausted");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
assert_eq!(outcome.counters.processed, 1);
assert_eq!(outcome.counters.attempt_failures, 4);
assert_eq!(task.get_progress().await.objects_scanned, 1);
assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]);
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
}
#[tokio::test]
async fn listing_failure_preserves_processed_objects_and_partial_coverage() {
let storage = Arc::new(MockStorage {
fail_second_listing_page: true,
..Default::default()
});
let task = bucket_task(storage);
task.execute().await.expect_err("second page cannot be traversed");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
assert_eq!(outcome.counters.processed, 1);
assert_eq!(outcome.objects[0].identity.object, "object-a");
assert_eq!(task.get_progress().await.objects_scanned, 1);
}
#[tokio::test]
async fn cluster_preserves_cumulative_progress_across_buckets() {
let storage = Arc::new(MockStorage {
list_each_bucket: true,
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage,
);
task.execute().await.expect("both buckets complete");
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.processed, 4);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
let progress = task.get_progress().await;
assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4));
assert_eq!(
outcome
.objects
.iter()
.filter(|item| item.identity.bucket == "bucket-b")
.count(),
2
);
}
#[tokio::test(start_paused = true)]
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
let storage = Arc::new(MockStorage::default());
storage.heal_object_outcomes.lock().expect("outcomes").insert(
"object-a".to_string(),
(0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(),
);
let task = bucket_task(storage.clone());
task.execute().await.expect_err("legacy adapter retains batch failure");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1));
assert_eq!(outcome.counters.attempt_failures, 4);
let failed = outcome
.objects
.iter()
.find(|item| item.identity.object == "object-a")
.expect("failed object");
assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted));
let object_b_calls = {
let calls = storage.heal_object_calls.lock().expect("calls");
calls.iter().filter(|object| object.as_str() == "object-b").count()
};
assert_eq!(object_b_calls, 1);
let progress = task.get_progress().await;
assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
}
#[tokio::test(start_paused = true)]
async fn retry_success_counts_one_terminal_outcome() {
let storage = Arc::new(MockStorage::default());
storage
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
let task = bucket_task(storage);
task.execute().await.expect("retry should recover");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.counters.processed, 2);
assert_eq!(outcome.counters.failed, 0);
assert_eq!(outcome.counters.attempt_failures, 1);
assert_eq!(
outcome
.objects
.iter()
.filter(|item| item.identity.object == "object-a")
.count(),
1
);
assert_eq!(
outcome.counters.processed,
outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed
);
}
#[tokio::test]
async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() {
let storage = Arc::new(MockStorage::default());
storage
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]));
let task = bucket_task(storage);
task.execute().await.expect("grace permits traversal completion");
let outcome = task.get_outcome().await;
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!(outcome.counters.processed, 2);
assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt");
assert!(matches!(
outcome.objects[0].disposition,
HealObjectDisposition::Deferred {
reason: HealDeferredReason::DanglingDeleteGrace,
..
}
));
assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown);
assert!(
outcome
.objects
.iter()
.all(|item| item.identity.bucket_incarnation_id.is_none())
);
assert_eq!(
task.get_progress().await.objects_healed,
1,
"legacy display count remains distinct from proof"
);
}
#[tokio::test]
async fn grace_single_object_is_completed_but_deferred() {
let storage = Arc::new(MockStorage {
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)),
..Default::default()
});
let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage);
task.execute().await.expect("grace is deferred");
let outcome = task.get_outcome().await;
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
assert_eq!(outcome.counters.processed, 1);
assert!(matches!(
outcome.objects[0].disposition,
HealObjectDisposition::Deferred {
reason: HealDeferredReason::DanglingDeleteGrace,
..
}
));
assert_eq!(outcome.counters.attempt_failures, 1);
}
#[tokio::test]
async fn dry_run_and_transient_existence_do_not_prove_repair() {
for transient in [false, true] {
let storage = Arc::new(MockStorage::default());
if transient {
storage
.object_exists_by_name
.lock()
.expect("existence fixture")
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
}
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
request.options.dry_run = !transient;
let task = HealTask::from_request(request, storage);
task.execute().await.expect("observation may complete");
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.healed, 0);
if transient {
assert!(matches!(
outcome.objects[0].disposition,
HealObjectDisposition::Deferred {
reason: HealDeferredReason::TransientExistenceCheck,
..
}
));
} else {
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
}
}
}
#[tokio::test]
async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage.clone(),
);
task.execute().await.expect_err("structural bucket error");
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
assert_eq!(
storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
["bucket-a", "bucket-b"]
);
}
#[tokio::test(start_paused = true)]
async fn cancellation_and_deadline_leave_partial_coverage() {
for cancel in [false, true] {
let storage = Arc::new(MockStorage {
block_heal_object: Mutex::new(true),
..Default::default()
});
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
request.options.timeout = Some(Duration::from_secs(1));
let task = HealTask::from_request(request, storage);
if cancel {
task.cancel().await.expect("cancel request");
}
task.execute().await.expect_err("control interruption");
let outcome = task.get_outcome().await;
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
assert_eq!(
outcome.execution,
HealExecutionOutcome::Aborted(if cancel {
HealAbortReason::Cancelled
} else {
HealAbortReason::Deadline
})
);
}
}
#[tokio::test]
async fn decode_keeps_the_requested_pool_and_set() {
let storage = Arc::new(MockStorage::default());
let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string()));
request.options.pool_index = Some(2);
request.options.set_index = Some(3);
let task = HealTask::from_request(request, storage.clone());
task.execute().await.expect("decode fixture");
let pool_and_set = {
let options = storage.object_heal_opts.lock().expect("storage options");
(options[0].pool, options[0].set)
};
assert_eq!(pool_and_set, (Some(2), Some(3)));
let outcome = task.get_outcome().await;
let identity = &outcome.objects[0].identity;
assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3)));
assert_eq!(identity.version_id.as_deref(), Some("version-a"));
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
}
}
use crate::heal::storage::{HealListItem, HealObjectInfo};
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
@@ -582,6 +940,10 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
#[derive(Default)]
struct MockStorage {
listed: Mutex<bool>,
list_each_bucket: bool,
fail_second_listing_page: bool,
recoverable_second_page_failures: Mutex<Option<usize>>,
listing_tokens: Mutex<Vec<Option<String>>>,
healed_objects: Mutex<Vec<String>>,
heal_object_calls: Mutex<Vec<String>>,
heal_object_version_ids: Mutex<Vec<Option<String>>>,
@@ -995,12 +1357,41 @@ impl HealStorageAPI for MockStorage {
_include_lifecycle_object_info: bool,
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
self.listed_prefixes.lock().unwrap().push(prefix.to_string());
self.listing_tokens
.lock()
.expect("listing tokens")
.push(continuation_token.map(ToOwned::to_owned));
if let Some(remaining) = self
.recoverable_second_page_failures
.lock()
.expect("listing failures")
.as_mut()
{
if continuation_token.is_none() {
return Ok((vec![heal_item("object-a")], Some("second".to_string()), true));
}
if *remaining > 0 {
*remaining -= 1;
return Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
bucket.to_string(),
"page".to_string(),
)));
}
return Ok((vec![heal_item("object-b")], None, false));
}
if self.fail_second_listing_page {
return if continuation_token.is_none() {
Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true))
} else {
Err(Error::other("listing unavailable"))
};
}
if *self.truncate_without_token.lock().unwrap() {
return Ok((vec![heal_item("object-a")], None, true));
}
let mut listed = self.listed.lock().unwrap();
if continuation_token.is_none() && !*listed {
if continuation_token.is_none() && (!*listed || self.list_each_bucket) {
*listed = true;
let objects = if bucket == RUSTFS_META_BUCKET {
vec![
@@ -1393,6 +1784,8 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
#[tokio::test]
async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
// A version listing can report the final page as truncated with no
// continuation token. That is treated as end-of-listing (not an error),
// so the returned page is healed and the pass terminates cleanly instead
@@ -1414,10 +1807,16 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
);
let task = HealTask::from_request(request, storage.clone());
task.heal_bucket("bucket-a")
task.execute()
.await
.expect("truncated-without-token must terminate cleanly, not loop or error");
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!(outcome.counters.processed, 1);
assert_eq!(
storage.healed_objects.lock().unwrap().as_slice(),
["object-a".to_string()],
+20 -4
View File
@@ -1888,9 +1888,9 @@ where
// A remote restart or movement flip invalidates
// the token proof; usage_store interprets this
// as a publication barrier and performs no PUT.
return true;
return Some(ScannerCycleDeferReason::DataMovement);
}
storeapi.scanner_data_usage_publication_blocked().await
scanner_local_publication_defer_reason(storeapi.as_ref()).await
}
},
)
@@ -3239,8 +3239,8 @@ where
{
match status {
ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
if storeapi.scanner_data_usage_publication_blocked().await {
return Some(ScannerCycleDeferReason::DataMovement);
if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await {
return Some(reason);
}
if status == ScannerCycleStatus::Complete {
let distributed = storeapi.setup_is_dist_erasure().await;
@@ -3263,6 +3263,22 @@ where
}
}
async fn scanner_local_publication_defer_reason<S>(storeapi: &S) -> Option<ScannerCycleDeferReason>
where
S: ScannerStorage,
{
if !storeapi.scanner_data_usage_publication_blocked().await {
return None;
}
// Pending namespace commits invalidate this publication attempt, but only
// storage movement creates durable, rate-limited catch-up debt.
if storeapi.scanner_data_movement_pause_status().await.paused {
Some(ScannerCycleDeferReason::DataMovement)
} else {
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
}
}
fn scanner_post_lease_activity_defer_reason(
expected_digest: Option<[u8; 32]>,
activity: Result<ScannerActivitySnapshot, String>,
+38 -12
View File
@@ -266,6 +266,11 @@ async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
}
let pause_status = store.scanner_data_movement_pause_status().await;
assert!(pause_status.paused);
assert_eq!(
scanner_local_publication_defer_reason(store.as_ref()).await,
Some(ScannerCycleDeferReason::DataMovement),
"an actual data-movement pause must retain durable catch-up tracking"
);
paused_probe.wait().await;
drop(paused_probe);
@@ -1171,6 +1176,9 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now())
.await
.expect("scanner pause backlog should be available");
let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default())
@@ -1195,6 +1203,13 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
.await
.expect("fixture usage baseline should be readable");
let pending = ecstore_hold_namespace_commit(store.as_ref());
assert_eq!(
scanner_local_publication_defer_reason(store.as_ref()).await,
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
"an ordinary namespace commit must not be classified as data movement"
);
let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
assert_eq!(pause_backlog_attempt, ScannerPauseBacklogAttemptDecision::Untracked);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default());
let mut cycle_info = CurrentCycle {
@@ -1209,7 +1224,15 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin
.await
.expect("the coordinator must finish its namespace walk while a PUT is pending");
assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal");
assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(
outcome,
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
finish_scanner_pause_backlog_cycle(&mut pause_backlog, &store, pause_backlog_attempt, outcome).await;
let pause_backlog_status = scanner_pause_backlog_status(store.clone()).await;
assert_eq!(pause_backlog_status.phase, ScannerPauseBacklogPhase::Idle);
assert!(!pause_backlog_status.pending_full_scan);
assert_eq!(pause_backlog_status.catch_up_attempts, 0);
assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle");
assert_eq!(revision, DataUsageCacheRevision::Missing);
assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before);
@@ -5826,7 +5849,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier
let probe_calls = route_probe_calls.clone();
async move {
let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
route_blocked && call > 1
(route_blocked && call > 1).then_some(ScannerCycleDeferReason::DataMovement)
}
},
)
@@ -5872,16 +5895,19 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert_eq!(
outcome,
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert!(!store.objects.lock().await.contains_key(&target_key));
assert_eq!(
store.put_counts.lock().await.get(&target_key),
None,
"the final pool-state fence must run before the first PUT"
"the final publication fence must run before the first PUT"
);
}
}
@@ -5902,7 +5928,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing(
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -5950,7 +5976,7 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -5991,7 +6017,7 @@ async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() {
receiver,
None,
None,
|| async { false },
|| async { None },
)
.await;
@@ -6051,7 +6077,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() {
data: Some(Bytes::from(snapshot_data)),
revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::DataMovement) },
)
.await;
@@ -6091,7 +6117,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() {
// Model the remote lease holder flipping its movement generation
// after the activity probe but before the coordinator's PUT.
route_store.publication_admission_blocked.store(true, Ordering::Release);
false
None
}
},
)
@@ -6129,7 +6155,7 @@ async fn coordinator_classifies_an_expired_publication_lease() {
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(None, Some(expired), None),
|| async { false },
|| async { None },
)
.await;
@@ -6208,7 +6234,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
|| async { true },
|| async { Some(ScannerCycleDeferReason::DataMovement) },
)
.await;
+20 -18
View File
@@ -265,7 +265,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
receiver,
leader_epoch,
initial_baseline,
|| async { false },
|| async { None },
)
.await
}
@@ -280,7 +280,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx,
@@ -308,7 +308,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx,
@@ -336,7 +336,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
Fut: Future<Output = Option<ScannerCycleDeferReason>> + Send,
{
let ScannerPublicationFence {
expected_publication_epoch,
@@ -374,18 +374,19 @@ where
} else {
DATA_USAGE_OBJ_NAME_PATH.as_str()
};
if route_probe().await {
if let Some(reason) = route_probe().await {
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_blocked_before_reconcile",
"Scanner data usage publication deferred by the pool-state fence"
reason = reason.as_str(),
path = %target_path,
"Scanner data usage publication deferred by the publication fence"
);
global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str());
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
global_metrics().record_scanner_usage_deferred(reason.as_str());
outcome = DataUsagePersistOutcome::Deferred(reason);
break;
}
@@ -626,17 +627,18 @@ where
if ctx.is_cancelled() {
break 'updates;
}
if route_probe().await {
if let Some(reason) = route_probe().await {
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_blocked_before_save",
"Scanner data usage publication deferred by the final pool-state fence"
reason = reason.as_str(),
path = %target_path,
"Scanner data usage publication deferred by the final publication fence"
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break DataUsagePersistOutcome::Deferred(reason);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
@@ -722,19 +724,19 @@ where
);
}
Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
let route_blocked = route_probe().await;
if route_blocked {
if let Some(reason) = route_probe().await {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %target_path,
state = "publication_deferred",
reason = reason.as_str(),
path = %target_path,
error = %e,
"Scanner data usage route is blocked by data movement; retrying later"
"Scanner data usage route remains blocked; retrying later"
);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break DataUsagePersistOutcome::Deferred(reason);
}
error!(
target: "rustfs::scanner",
+2
View File
@@ -1294,6 +1294,8 @@ impl FolderScanner {
}
Err(e) => return Err(ScannerError::Io(e)),
};
#[cfg(test)]
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
@@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::Mutex;
mod checkpoint_fixture;
pub(super) mod enumeration_restart;
/// Reset the process-global alert cooldown map; test-only.
fn reset_alert_cooldowns() {
@@ -20,6 +20,8 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
mod segment_observation;
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
const STATIC_OBJECTS: u64 = 24;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
@@ -0,0 +1,202 @@
//! Fixture-only range diagnostics. No result is supplied to a scan selector.
use super::*;
use std::collections::BTreeSet;
const MAX_SEGMENTS: usize = 4;
const MAX_SEGMENT_BYTES: usize = 128;
const MAX_WALK_SAMPLES: usize = 32;
const MAX_WALK_BYTES: usize = 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ProposalError {
EntryLimit,
ByteLimit,
InvalidKey,
}
// Keys come from successful fixture writes, not a production mutation stream.
fn fixture_proposal(keys: &[&str]) -> Result<BTreeSet<String>, ProposalError> {
let mut segments = BTreeSet::new();
let mut bytes = 0;
for key in keys {
if key.is_empty() || key.contains(['\\', '\0']) || key.split('/').any(|part| matches!(part, "" | "." | "..")) {
return Err(ProposalError::InvalidKey);
}
let segment = key.split('/').next().expect("validated nonempty key");
if segments.contains(segment) {
continue;
}
if segments.len() == MAX_SEGMENTS {
return Err(ProposalError::EntryLimit);
}
if segment.len() > MAX_SEGMENT_BYTES - bytes {
return Err(ProposalError::ByteLimit);
}
bytes += segment.len();
segments.insert(segment.to_string());
}
Ok(segments)
}
#[test]
fn segment_observation_fixture_proposal_bounds() {
assert_eq!(fixture_proposal(&["hot/one", "hot/two"]), Ok(BTreeSet::from(["hot".to_string()])));
assert_eq!(fixture_proposal(&["a", "b", "c", "d"]).expect("entry boundary").len(), MAX_SEGMENTS);
assert_eq!(fixture_proposal(&["a", "b", "c", "d", "e"]), Err(ProposalError::EntryLimit));
let exact = "x".repeat(MAX_SEGMENT_BYTES);
assert!(fixture_proposal(&[&exact]).is_ok());
assert_eq!(fixture_proposal(&[&exact, "y"]), Err(ProposalError::ByteLimit));
let oversized = "x".repeat(MAX_SEGMENT_BYTES + 1);
assert_eq!(fixture_proposal(&[&oversized]), Err(ProposalError::ByteLimit));
for key in ["", "/hot", "hot/../cold", "hot//one", "hot\\one", "hot/\0"] {
assert_eq!(fixture_proposal(&[key]), Err(ProposalError::InvalidKey));
}
}
fn cache_value(cache: &DataUsageCache) -> serde_json::Value {
let mut value = serde_json::to_value(cache).expect("serialize the entire cache");
// Children are a HashSet: canonicalize only that unordered field, without
// discarding any cache fields or changing ordered histogram arrays.
for (path, entry) in &cache.cache {
value["cache"][path]["children"] =
serde_json::to_value(entry.children.iter().collect::<BTreeSet<_>>()).expect("canonical child set");
}
value
}
async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
let (mut scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for prefix in ["hot", "cold", "other"] {
for leaf in ["one", "two"] {
let object = format!("{prefix}/{leaf}");
let mut metadata = FileMeta::new();
let mut info = FileInfo::new(&object, 4, 2);
info.volume = "bucket".to_string();
info.name = object.clone();
info.size = 1;
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
info.metadata.insert("etag".to_string(), "before".to_string());
metadata.add_version(info).expect("construct segment fixture metadata");
write_test_object_metadata_bytes(&root, "bucket", &object, &metadata.marshal_msg().expect("encode metadata")).await;
}
}
let changed_key = "hot/one";
let changed_path = root.join("bucket").join(changed_key).join("xl.meta");
let before = tokio::fs::read(&changed_path).await.expect("read initial hot metadata");
let mut metadata = FileMeta::new();
let mut info = FileInfo::new(changed_key, 4, 2);
info.volume = "bucket".to_string();
info.name = changed_key.to_string();
info.size = 1;
info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp"));
info.metadata.insert("etag".to_string(), "after!".to_string());
metadata.add_version(info).expect("construct same-size hot mutation");
write_test_object_metadata_bytes(&root, "bucket", changed_key, &metadata.marshal_msg().expect("encode hot mutation")).await;
let after = tokio::fs::read(&changed_path)
.await
.expect("read back committed fixture mutation");
assert_eq!(before.len(), after.len(), "fixture rewrite must keep metadata byte length unchanged");
assert_ne!(before, after, "a changed key requires an observable successful fixture write");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
let paths = Arc::new(Mutex::new(Vec::<String>::new()));
let proposed_walked = Arc::new(Mutex::new(BTreeSet::<String>::new()));
scanner.update_current_path = Arc::new({
let paths = paths.clone();
let proposed_walked = proposed_walked.clone();
move |path: &str| {
let mut paths = paths.lock().expect("lock bounded actual-walk samples");
assert!(paths.len() < MAX_WALK_SAMPLES, "fixture walk exceeded its entry budget");
let bytes: usize = paths.iter().map(String::len).sum();
assert!(path.len() <= MAX_WALK_BYTES - bytes, "fixture walk exceeded its byte budget");
paths.push(path.to_string());
if observe {
let proposed = fixture_proposal(&[changed_key]).expect("bounded successful fixture mutation");
if let Some(segment) = path.strip_prefix("bucket/").and_then(|path| path.split('/').next())
&& proposed.contains(segment)
{
proposed_walked
.lock()
.expect("lock bounded observed segments")
.insert(segment.to_string());
}
}
Box::pin(async {})
}
});
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
},
&mut DataUsageEntry::default(),
)
.await
.expect("actual folder walker must finish independently of diagnostics");
let paths = paths.lock().expect("read walk samples").clone();
assert!(!paths.is_empty());
for prefix in ["hot", "cold", "other"] {
assert!(
paths.iter().any(|path| path == &format!("bucket/{prefix}")),
"all fixture segments must actually be walked"
);
}
let store = FixtureStore::new();
let revisions = DataUsageCache::default()
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("read empty fixture revisions");
scanner
.new_cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("save actual walker output through the cache codec and revision gate");
let loaded = store.strict_load().await;
assert_eq!(loaded.checked_flatten("bucket").expect("complete fixture tree").objects, 6);
assert_eq!(
cache_value(&loaded),
cache_value(&scanner.new_cache),
"codec round-trip must retain the entire cache, not just aggregate size"
);
if observe {
let proposed = proposed_walked.lock().expect("read callback observations").clone();
assert_eq!(proposed, BTreeSet::from(["hot".to_string()]));
let walked_segments: BTreeSet<_> = paths
.iter()
.filter_map(|path| path.strip_prefix("bucket/"))
.filter_map(|path| path.split('/').next())
.collect();
assert_eq!(walked_segments, BTreeSet::from(["cold", "hot", "other"]));
assert!(proposed.iter().all(|segment| walked_segments.contains(segment.as_str())));
assert_eq!(
walked_segments.len() - proposed.len(),
2,
"the two non-proposed segments must still be walked"
);
eprintln!(
"segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified",
paths.len()
);
} else {
assert!(proposed_walked.lock().expect("read disabled observations").is_empty());
}
// Compare semantic values because map encoding order is not content identity.
(paths, cache_value(&loaded))
}
#[tokio::test]
#[serial]
async fn segment_observation_on_off_preserves_actual_walk_and_saved_cache() {
let off = walk_and_save(false).await;
let on = walk_and_save(true).await;
assert_eq!(off.0, on.0, "diagnostics must not change actual traversal order or coverage");
assert_eq!(off.1, on.1, "diagnostics must not change the saved cache result");
}
@@ -0,0 +1,181 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use std::path::{Path, PathBuf};
use tokio::io::AsyncReadExt;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST";
struct Observation {
root: PathBuf,
limit: u64,
entries: u64,
name_bytes: u64,
}
static OBSERVATION: Mutex<Option<Observation>> = Mutex::new(None);
// Only the selected synthetic disk is observed; concurrent unrelated scanners
// do not consume its budget. This hook is absent from non-test builds.
pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) {
let mut guard = OBSERVATION.lock().expect("enumeration observation lock");
if let Some(observation) = guard.as_mut()
&& Path::new(dir).starts_with(&observation.root)
{
observation.entries += 1;
observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name");
if observation.entries >= observation.limit {
budget.cancel_for_runtime();
}
}
}
struct ObservationGuard;
impl Drop for ObservationGuard {
fn drop(&mut self) {
*OBSERVATION.lock().expect("enumeration observation cleanup") = None;
}
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct Request {
workspace: PathBuf,
objects: usize,
raw_entry_budget: u64,
round: u32,
}
async fn read_bounded(path: &Path) -> Vec<u8> {
let file = tokio::fs::File::open(path).await.expect("open fixture artifact");
let mut bytes = Vec::new();
file.take(MAX_CACHE_BYTES + 1)
.read_to_end(&mut bytes)
.await
.expect("read fixture artifact");
assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES);
bytes
}
async fn round(request: &Request) -> serde_json::Value {
assert!((1..=1024).contains(&request.objects));
assert!((1..=4096).contains(&request.raw_entry_budget));
assert!(request.round < 64);
let disk_root = request.workspace.join("disk");
let cache_path = request.workspace.join("cache.bin");
if request.round == 0 {
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
for index in 0..request.objects {
let object = format!("object-{index:04}");
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
let bytes = metadata_for_object_version("bucket", &object, Some(version));
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
}
let mut initial = DataUsageCache::default();
initial.info.name = "bucket".to_string();
initial.info.skip_healing = true;
initial.info.snapshot_complete = false;
initial.replace("bucket", "", DataUsageEntry::default());
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
.await
.expect("persist initial cache");
}
let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan");
assert_eq!(cache.info.name, "bucket");
let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects;
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open synthetic disk in this process");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
*OBSERVATION.lock().expect("install observation") = Some(Observation {
root: disk.path(),
limit: request.raw_entry_budget,
entries: 0,
name_bytes: 0,
});
let _observation_guard = ObservationGuard;
let result = scan_data_folder(
budget.token(),
budget.clone(),
vec![disk.clone()],
disk,
cache.clone(),
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
let (returned, outcome) = match result {
Ok(cache) => (cache, "complete"),
Err(ScannerError::PartialCache(cache)) => (*cache, "partial"),
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {
(cache, "cancelled_without_cache")
}
Err(error) => panic!("unexpected real scanner failure: {error}"),
};
let encoded = returned.marshal_msg().expect("returned cache codec");
assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES);
tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache");
let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec");
let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root");
let scanned = returned.checked_flatten("bucket").expect("returned bucket root");
assert_eq!(
(retained.objects, retained.versions, retained.size),
(scanned.objects, scanned.versions, scanned.size)
);
assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete);
let guard = OBSERVATION.lock().expect("read observation");
let observation = guard.as_ref().expect("installed observation");
serde_json::json!({
"schema": 1, "pid": std::process::id(), "round": request.round,
"objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget,
"raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes,
"objects_processed": budget.progress().0,
"objects_before": before, "objects_retained": retained.objects,
"versions_retained": retained.versions, "bytes_retained": retained.size,
"snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome,
})
}
/// Default CI is a positive healthy control. The external driver selects the
/// same worker in a fresh OS process per round and applies its strict oracle.
#[tokio::test]
#[serial]
async fn enumeration_restart_worker() {
if let Some(path) = std::env::var_os(REQUEST_ENV) {
let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request");
let report = round(&request).await;
tokio::fs::write(
request.workspace.join(format!("round-{}.json", request.round)),
serde_json::to_vec(&report).expect("report JSON"),
)
.await
.expect("write worker report");
} else {
let temp = tempfile::tempdir().expect("healthy fixture directory");
let report = round(&Request {
workspace: temp.path().to_path_buf(),
objects: 4,
raw_entry_budget: 16,
round: 0,
})
.await;
assert_eq!(report["outcome"], "complete");
assert_eq!(report["snapshot_complete"], true);
assert_eq!(report["objects_retained"], 4);
assert_eq!(report["versions_retained"], 4);
assert_eq!(report["bytes_retained"], 4);
assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}");
}
}
+12
View File
@@ -124,6 +124,16 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
(temp_dir, store)
}
async fn wait_for_namespace_commit_tails(store: &ECStore) {
tokio::time::timeout(Duration::from_secs(30), async {
while store.scanner_data_usage_publication_blocked().await {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("namespace commit tails should drain before the scanner fixture runs");
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_bucket_identity_uses_its_set_instance_owner() {
@@ -334,6 +344,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work()
.await
.expect("initial object should persist");
}
wait_for_namespace_commit_tails(store.as_ref()).await;
let mut baseline = None;
for (index, (scan_mode, requires_full_scan, explicit_scope)) in [
(HealScanMode::Normal, true, false),
@@ -352,6 +363,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work()
.put_object("cold-bucket", &format!("added-{index}"), &mut reader, &ScannerObjectOptions::default())
.await
.expect("cold bucket mutation should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
// Only the hot bucket is in the usage hint. The cold result must
// come from this cycle's storage walk, not its previous baseline.
record_dirty_usage_bucket("hot-bucket");