diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json index 6cadc7815..24b723e39 100644 --- a/.config/ecstore-required-tests.json +++ b/.config/ecstore-required-tests.json @@ -40,6 +40,21 @@ "invariant": "corrupt-part-arrays", "suite": "rustfs-filemeta", "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics" + }, + { + "invariant": "odm-source-contract-s3", + "suite": "rustfs", + "name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract" + }, + { + "invariant": "odm-source-contract-azure", + "suite": "rustfs", + "name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract" + }, + { + "invariant": "odm-source-contract-gcs", + "suite": "rustfs", + "name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract" } ], "fixtures": [ diff --git a/.config/scanner-heal-required-tests.json b/.config/scanner-heal-required-tests.json new file mode 100644 index 000000000..abf25420c --- /dev/null +++ b/.config/scanner-heal-required-tests.json @@ -0,0 +1,40 @@ +{ + "schema": 1, + "cases": { + "background-target-restart": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart", + "oracle": "background-target-restart.json", + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 4, "drives_per_node": 1}, + "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4." + } + }, + "release_pending": { + "G01": "W02/W04 complete root and quota authority coverage", + "G02": "W03 bounded checkpoint progress and independent version inventory", + "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers", + "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary", + "G05": "W06/W07 per-object outcomes and bounded terminal retention", + "G06": "W06/W08/W23 concurrent status, legacy clients and truncation", + "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary", + "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix", + "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads", + "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery", + "G11": "W04/W19/W24 maintenance and complete producer coverage", + "G12": "W02/W15/W16 both quota paths during reset and settlement", + "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail", + "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage", + "P1": "W20 measured cold-walk share and foreground latency/throughput", + "P2": "W20/W24 measured post-stop convergence and cold segment reuse", + "P3": "W20 measured two-hour pressure/heal capacity and recovery window", + "P4": "W20 measured MRF scale and replay cost with retained responsibility", + "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification", + "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace", + "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement" + } +} diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml index 187defc32..c3a5ca0e1 100644 --- a/.github/workflows/e2e-distributed.yml +++ b/.github/workflows/e2e-distributed.yml @@ -20,6 +20,13 @@ # `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs, # nightly runs, and manual dispatches all execute the same fail-closed suite. # Upgrade cases download the same pinned previous release as e2e-upgrade.yml. +# +# Isolated pool filesystems: expand/decommission/rebalance cases require +# independent `statfs` capacity. `sm-standard-4` is an ARC pod +# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no +# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed: +# No such file or directory"). The prepare step therefore mounts four 1 GiB +# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`. name: e2e-distributed @@ -101,21 +108,21 @@ jobs: mkdir -p "${mount_base}" roots=() for pool in 0 1 2 3; do - image="${mount_base}/pool-${pool}.img" mountpoint="${mount_base}/pool-${pool}" - truncate -s 1G "${image}" - mkfs.ext4 -q -F "${image}" mkdir -p "${mountpoint}" - sudo mount -o loop,nosuid,nodev "${image}" "${mountpoint}" + # sm-standard-4 is an ARC pod without usable loop devices, so + # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a + # distinct st_dev and independent 1G statfs capacity. + sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}" sudo chmod 1777 "${mountpoint}" roots+=("${mountpoint}") done printf -v joined_roots '%s:' "${roots[@]}" echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}" - findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[0]}" - findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[1]}" - findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[2]}" - findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[3]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}" - name: Download pinned previous release env: diff --git a/crates/e2e_test/build.rs b/crates/e2e_test/build.rs new file mode 100644 index 000000000..6d412d3d4 --- /dev/null +++ b/crates/e2e_test/build.rs @@ -0,0 +1,74 @@ +// Copyright 2024 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use std::path::Path; +use std::process::Command; + +fn git(root: &Path, args: &[&str]) -> Option { + let output = Command::new("git").args(args).current_dir(root).output().ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn emit(name: &str, value: &str) { + let value = if value.contains(['\n', '\r']) { "unknown" } else { value }; + println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}"); +} + +fn main() { + let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default(); + let root = Path::new(&manifest).join("../.."); + // Cover dependency/common sources as well as this crate. HEAD/ref/index + // changes must refresh identity even when no Rust source mtime changes. + for path in [ + "crates", + "rustfs", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + ".cargo", + ".config", + ] { + println!("cargo:rerun-if-changed={}", root.join(path).display()); + } + let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()]; + if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) { + git_paths.push(reference); + } + for path in git_paths { + if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) { + let path = Path::new(&path); + let path = if path.is_absolute() { + path.to_owned() + } else { + root.join(path) + }; + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + } + } + } + let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned()); + let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty()); + let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned()); + let mut features = std::env::vars() + .filter_map(|(key, _)| { + key.strip_prefix("CARGO_FEATURE_") + .map(|name| name.to_ascii_lowercase().replace('_', "-")) + }) + .collect::>(); + features.sort(); + emit("COMMIT", &revision); + emit("DIRTY", if dirty { "true" } else { "false" }); + emit("LOCK", &lock); + emit("FEATURES", &features.join(",")); + for name in ["TARGET", "PROFILE"] { + emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned())); + } + println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS"); + let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default(); + let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect(); + emit("RUSTFLAGS_HEX", &flags); +} diff --git a/crates/e2e_test/src/chaos.rs b/crates/e2e_test/src/chaos.rs index 920a87f0c..f0b2971e9 100644 --- a/crates/e2e_test/src/chaos.rs +++ b/crates/e2e_test/src/chaos.rs @@ -55,18 +55,20 @@ type ChaosResult = Result>; /// A successful S3 GET only proves that a quorum can serve an object. Replacement /// tests need this lower-level record to prove that the rebuilt target holds the /// `xl.meta` selected for a specific version and every `part.N` it declares. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub(crate) struct VersionShardCensus { pub version_id: Option, pub has_xl_meta: bool, pub data_dir: Option, pub erasure_index: Option, + pub data_blocks: Option, + pub parity_blocks: Option, pub expected_part_numbers: BTreeSet, pub present_part_fingerprints: BTreeMap, pub inline_data_fingerprint: Option, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub(crate) struct PartShardFingerprint { pub size: u64, pub sha256: String, @@ -88,13 +90,15 @@ impl VersionShardCensus { && manifest.is_complete() && self.data_dir == manifest.data_dir && self.erasure_index == manifest.erasure_index + && self.data_blocks == manifest.data_blocks + && self.parity_blocks == manifest.parity_blocks && self.expected_part_numbers == manifest.expected_part_numbers && self.present_part_fingerprints == manifest.present_part_fingerprints && self.inline_data_fingerprint == manifest.inline_data_fingerprint } } -fn sha256_hex(data: &[u8]) -> String { +pub(crate) fn sha256_hex(data: &[u8]) -> String { let digest = Sha256::digest(data); digest.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: false, data_dir: None, erasure_index: None, + data_blocks: None, + parity_blocks: None, expected_part_numbers: BTreeSet::new(), present_part_fingerprints: BTreeMap::new(), inline_data_fingerprint: None, @@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: true, data_dir, erasure_index, + data_blocks: Some(file_info.erasure.data_blocks), + parity_blocks: Some(file_info.erasure.parity_blocks), expected_part_numbers, present_part_fingerprints, inline_data_fingerprint, @@ -413,6 +421,8 @@ mod tests { has_xl_meta: true, data_dir: Some("data-dir".to_string()), erasure_index: Some(3), + data_blocks: Some(2), + parity_blocks: Some(2), expected_part_numbers: BTreeSet::from([1]), present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]), inline_data_fingerprint: None, diff --git a/crates/e2e_test/src/data_usage_test.rs b/crates/e2e_test/src/data_usage_test.rs index 141ab29c7..512d5e16c 100644 --- a/crates/e2e_test/src/data_usage_test.rs +++ b/crates/e2e_test/src/data_usage_test.rs @@ -35,11 +35,15 @@ where { let mut last_usage = DataUsageInfo::default(); let mut last_query_error = None; - for _ in 0..45 { + for _ in 0..90 { match get_data_usage_info(env).await { Ok(usage) => { last_query_error = None; - if usage.buckets_usage.contains_key(bucket) && predicate(&usage) { + if usage.is_complete_bucket_usage_snapshot() + && usage.usage_snapshot_converged != Some(false) + && usage.buckets_usage.contains_key(bucket) + && predicate(&usage) + { return Ok(usage); } last_usage = usage; diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index 8201210ec..68b0006b1 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -16,15 +16,18 @@ #[cfg(test)] mod tests { - use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post}; + use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post}; use crate::common::{ FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, + rustfs_binary_path, }; use crate::storage_api::RUSTFS_META_BUCKET; use aws_sdk_s3::primitives::ByteStream; use http::Method; + use sha2::{Digest, Sha256}; use std::collections::HashSet; use std::error::Error; + use std::io::{Read, Write}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Command; @@ -34,6 +37,76 @@ mod tests { const POOL_METADATA_OBJECT: &str = "pool.bin"; + #[derive(serde::Deserialize)] + struct EvidenceBuild { + sha256: String, + } + + #[derive(serde::Deserialize)] + struct RestartEvidenceRun { + schema: u32, + run_id: String, + source_revision: String, + test_build: serde_json::Value, + binary: EvidenceBuild, + test_binary: EvidenceBuild, + } + + fn file_sha256(path: &Path) -> Result> { + let mut file = std::fs::File::open(path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect()) + } + + fn restart_evidence_run(binary: &Path) -> Result, Box> { + let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else { + return Ok(None); + }; + let directory = PathBuf::from(directory); + let receipt = directory.join("run.json"); + if receipt.metadata()?.len() > 1024 * 1024 { + return Err("oversized scanner/heal execution receipt".into()); + } + let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?; + if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 { + return Err("invalid scanner/heal execution identity".into()); + } + let built = compiled_test_identity(); + for key in ["source_revision", "dirty", "lock_blob", "features"] { + assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}"); + } + assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt"); + assert_eq!( + file_sha256(&std::env::current_exe()?)?, + run.test_binary.sha256, + "test executable must match the run receipt" + ); + if directory.join("background-target-restart.json").exists() { + return Err("scanner/heal oracle already exists; create a new execution receipt".into()); + } + Ok(Some((directory, run))) + } + + fn compiled_test_identity() -> serde_json::Value { + serde_json::json!({ + "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"), + "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false", + "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"), + "features": env!("RUSTFS_E2E_BUILD_FEATURES"), + "target": env!("RUSTFS_E2E_BUILD_TARGET"), + "profile": env!("RUSTFS_E2E_BUILD_PROFILE"), + "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"), + }) + } + struct TcpPortBlackhole { port: u16, comment: String, @@ -195,8 +268,9 @@ mod tests { clients: &[aws_sdk_s3::Client], bucket: &str, expected_keys: &HashSet, - ) -> Result<(), Box> { + ) -> Result>, Box> { const PAGE_SIZE: i32 = 10; + let mut node_listings = Vec::with_capacity(clients.len()); for (node_index, client) in clients.iter().enumerate() { let mut listed_keys = Vec::new(); let mut continuation_token = None; @@ -243,8 +317,10 @@ mod tests { &listed_key_set, expected_keys, "node {node_index} did not expose the complete recovered namespace" ); + listed_keys.sort(); + node_listings.push(listed_keys); } - Ok(()) + Ok(node_listings) } fn heal_task_status_diagnostic(body: &str) -> String { @@ -808,6 +884,13 @@ mod tests { } async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box> { + let server_binary = rustfs_binary_path(); + let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart { + restart_evidence_run(&server_binary)? + } else { + None + }; + let mut evidence_objects = Vec::new(); let (background_enabled, interruption_node, interruption_kind) = match scenario { InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"), InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"), @@ -855,7 +938,7 @@ mod tests { for node_index in 0..cluster.nodes.len() { cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?; } - cluster.start().await?; + cluster.start_with_binary(&server_binary).await?; let clients = cluster.create_all_clients()?; let bucket = "heal-restart-during-rebuild"; @@ -996,7 +1079,7 @@ mod tests { } } - cluster.start_node(1).await?; + cluster.start_node_from_binary(1, &server_binary).await?; let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url); let recovery_deadline = Instant::now() + Duration::from_secs(60); @@ -1274,7 +1357,7 @@ mod tests { } } } - cluster.start_node(interruption_node).await?; + cluster.start_node_from_binary(interruption_node, &server_binary).await?; if interruption_node == 0 { let target = cluster.nodes[1] .process @@ -1373,7 +1456,7 @@ mod tests { .map(|manifest| manifest.key.clone()) .collect::>(); assert!(expected_keys.insert(outage_key.to_string())); - assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?; + let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?; let target_client = cluster.create_s3_client(1)?; for expected in &expected_manifests { @@ -1381,11 +1464,31 @@ mod tests { let actual = response.body.collect().await?.into_bytes(); let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed); assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key); + if evidence_run.is_some() { + evidence_objects.push(serde_json::json!({ + "key": expected.key, "version_id": expected.shard_census.version_id, + "expected_bytes": expected_body.len(), "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(&expected_body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": expected.shard_census, + "physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?, + })); + } } let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?; let actual = response.body.collect().await?.into_bytes(); let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed); assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}"); + if evidence_run.is_some() { + evidence_objects.push(serde_json::json!({ + "key": outage_key, "version_id": null, + "expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(&expected_outage_body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": null, + "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?, + })); + } let terminal_deadline = Instant::now() + Duration::from_secs(30); loop { @@ -1432,6 +1535,31 @@ mod tests { return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into()); } + if let Some((directory, run)) = evidence_run { + let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id(); + assert_ne!(target_pid, restarted_pid, "target must be a new process"); + assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart"); + let evidence = serde_json::json!({ + "schema": 1, "case": "background-target-restart", "evidence": "process-restart", + "run_id": run.run_id, "source_revision": run.source_revision, + "test_build": compiled_test_identity(), + "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256, + "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()}, + "pid_before": target_pid, "pid_after": restarted_pid, + "objects": evidence_objects, "node_listings": node_listings, + }); + let data = serde_json::to_vec(&evidence)?; + if data.len() > 1024 * 1024 { + return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into()); + } + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join("background-target-restart.json"))?; + output.write_all(&data)?; + output.sync_all()?; + } + Ok(()) } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 5d4bba0b9..26325e49f 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -32,7 +32,7 @@ pub mod bucket { pub mod bucket_target_sys { pub use crate::bucket::bucket_target_sys::{ AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, - SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query, + SsecPassthroughCapability, TargetClient, append_version_id_query, }; } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index ce8ec5a67..9cf4ed902 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -369,26 +369,6 @@ struct SsecPassthroughRecord { recorded_at: Instant, } -/// What a target write does when the bucket's persisted target set exists but -/// cannot be decoded. -/// -/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a -/// configuration that could not be fully read, because re-serializing a -/// partial in-memory view is the one mechanism by which a configured target -/// really disappears. That rule guards against an *unintentional* overwrite, -/// so an operator who names the hazard keeps a repair path -/// (rustfs/backlog#2309); everything that does not name it stays refused. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum UnreadableTargetsPolicy { - /// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`]. - #[default] - FailClosed, - /// Discard the unreadable set; the target being written becomes the whole - /// configuration. Reachable only from an admin request that asked for it - /// explicitly, and audited by the caller. - Replace, -} - #[derive(Debug, Default)] pub struct BucketTargetSys { pub arn_remotes_map: Arc>>, @@ -811,41 +791,23 @@ impl BucketTargetSys { bucket: &str, target: &BucketTarget, update: bool, - unreadable_policy: UnreadableTargetsPolicy, ) -> Result { self.validate_target(bucket, target).await?; - let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?; + let mut bucket_targets = self.targets_base_for_write(bucket).await?; Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?; Ok(bucket_targets) } - /// The persisted target set a write merges into. - /// - /// An absent configuration starts from the empty set. An unreadable one is - /// refused, because re-serializing a partial view of a set this node could - /// not decode is how a configured target disappears for good — unless the - /// caller carries the operator's explicit - /// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it - /// deliberately (rustfs/backlog#2309). - async fn targets_base_for_write( - &self, - bucket: &str, - unreadable_policy: UnreadableTargetsPolicy, - ) -> Result { + /// Ordinary writes must not turn an unreadable cached snapshot into an + /// empty configuration. Explicit repair belongs to the metadata transaction + /// that can inspect the current persisted state. + async fn targets_base_for_write(&self, bucket: &str) -> Result { match self.list_bucket_targets(bucket).await { Ok(targets) => Ok(targets), Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()), - // The opt-in discards only a set this node genuinely cannot read. - // A readable set still merges through the arm above, so the policy - // can never drop a target that was visible here. - Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) - if unreadable_policy == UnreadableTargetsPolicy::Replace => - { - Ok(BucketTargets::default()) - } Err(err) => Err(err), } } @@ -908,7 +870,9 @@ impl BucketTargetSys { Ok(()) } - fn upsert_target_entry( + /// Merge a validated target into a caller-owned snapshot. The caller must + /// protect that snapshot through persistence. + pub fn upsert_target_entry( bucket_targets: &mut Vec, target: &BucketTarget, update: bool, @@ -1272,25 +1236,27 @@ impl BucketTargetSys { return (String::new(), false); }; - { - let targets_map = self.targets_map.read().await; - if let Some(targets) = targets_map.get(bucket) { - for tgt in targets { - if tgt.target_type == target.target_type - && tgt.target_bucket == target.target_bucket - && target.endpoint == tgt.endpoint - && tgt - .credentials - .as_ref() - .map(|c| { - let default_creds = Credentials::default(); - c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key - }) - .unwrap_or(false) - { - return (tgt.arn.clone(), true); - } - } + let targets_map = self.targets_map.read().await; + let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default(); + Self::remote_arn_for_targets(targets, target, depl_id) + } + + /// Resolve create idempotency against the snapshot the caller will persist. + pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) { + for tgt in targets { + if tgt.target_type == target.target_type + && tgt.target_bucket == target.target_bucket + && target.endpoint == tgt.endpoint + && tgt + .credentials + .as_ref() + .map(|c| { + let default_creds = Credentials::default(); + c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key + }) + .unwrap_or(false) + { + return (tgt.arn.clone(), true); } } @@ -4371,47 +4337,19 @@ mod tests { } } - /// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable - /// `bucket-targets.json` left the bucket with no API repair path at all. - /// The refusal is the default and stays the default; the operator's - /// explicit opt-in is the only thing that discards the set, and it starts - /// the replacement from empty rather than from a partial view of bytes - /// this node never decoded. #[tokio::test] - async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() { + async fn an_unreadable_target_set_refuses_cached_writes() { let sys = BucketTargetSys::default(); let bucket = "targets-repair-opt-in"; sys.mark_targets_unreadable(bucket).await; - - assert!( - matches!( - sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await, - Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) - ), - "without the opt-in an unreadable target set must still refuse the write" - ); - assert_eq!( - UnreadableTargetsPolicy::default(), - UnreadableTargetsPolicy::FailClosed, - "a caller that says nothing must get the refusal" - ); - - let base = sys - .targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace) - .await - .expect("the explicit opt-in must let an operator replace an unreadable set"); - assert!( - base.is_empty(), - "the replacement must start from an empty set, never from a partial decode" - ); + assert!(matches!( + sys.targets_base_for_write(bucket).await, + Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) + )); } - /// The opt-in is not a wipe switch. On a set this node can read, both - /// policies take the same merge path, so a stray `replace-unreadable=true` - /// cannot drop a visible target — which is what makes the flag safe to - /// repeat in an operator's repair script. #[tokio::test] - async fn the_opt_in_never_discards_a_readable_target_set() { + async fn a_readable_target_set_remains_the_write_base() { let sys = BucketTargetSys::default(); let bucket = "targets-repair-readable"; let existing = repair_target(bucket, "keep"); @@ -4419,14 +4357,8 @@ mod tests { .write() .await .insert(bucket.to_string(), vec![existing.clone()]); - - for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] { - let base = sys - .targets_base_for_write(bucket, policy) - .await - .expect("a readable target set must be readable under either policy"); - assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target"); - assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target"); - } + let base = sys.targets_base_for_write(bucket).await.expect("read targets"); + assert_eq!(base.targets.len(), 1); + assert_eq!(base.targets[0].arn, existing.arn); } } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index ad2241629..2d90e9762 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp #[cfg(test)] static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity( obj_name: &str, rv_id: &str, @@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source( #[cfg(test)] mod test { + #[cfg(feature = "test-util")] + use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity; use rustfs_s3_client::signer_error::invalid_utf8_header_error; use super::{ CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry, RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity, - delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent, - delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error, - is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure, - transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, + delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity, + is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, + should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, }; use crate::storage_api_contracts::lifecycle::TransitionedObject; use rustfs_filemeta::TransitionVersionState; diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 3c866ae6f..426956d13 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -747,7 +747,7 @@ pub enum TransitionTransactionRecoveryOutcome { OperatorRequired(IlmRecoveryErrorCode), } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] #[derive(Default)] struct TransitionRecoveryClaimBarrierState { transaction_id: Uuid, @@ -755,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionRecoveryClaimBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl TransitionRecoveryClaimBarrier { pub(crate) fn install(transaction_id: Uuid) -> Self { let state = Arc::new(TransitionRecoveryClaimBarrierState { @@ -796,7 +796,7 @@ impl TransitionRecoveryClaimBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for TransitionRecoveryClaimBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -810,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_before_transition_recovery_claim(transaction_id: Uuid) { let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -825,7 +825,7 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] #[derive(Default)] struct TransitionRecoveryTerminalBarrierState { transaction_id: Uuid, @@ -833,17 +833,17 @@ struct TransitionRecoveryTerminalBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionRecoveryTerminalBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl TransitionRecoveryTerminalBarrier { pub(crate) fn install(transaction_id: Uuid) -> Self { let state = Arc::new(TransitionRecoveryTerminalBarrierState { @@ -870,7 +870,7 @@ impl TransitionRecoveryTerminalBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for TransitionRecoveryTerminalBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -884,7 +884,7 @@ impl Drop for TransitionRecoveryTerminalBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) { let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -1242,7 +1242,7 @@ async fn process_transition_transaction_record_at( }, ) .map_err(transition_transaction_store_error)?; - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pause_before_transition_recovery_claim(current.transaction_id).await; match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await { Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await, @@ -1300,7 +1300,7 @@ async fn process_transition_transaction_record_at( }; persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?; if let Some(source) = source_to_delete { - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pause_after_transition_recovery_terminal(source.transaction_id).await; delete_transition_transaction_record(api, &source).await?; } @@ -1322,7 +1322,7 @@ fn transition_recovery_control_identity(transaction: &TransitionTransaction, rec } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result { let record_name = transition_transaction_record_object_name(transaction.transaction_id)?; transition_recovery_control_identity(transaction, &record_name) @@ -1753,7 +1753,7 @@ pub async fn recover_transition_transaction_records( recover_transition_transaction_records_with_now(api, limit, marker, None).await } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub async fn recover_transition_transaction_records_at( api: Arc, limit: usize, diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index 4032773f5..fecb3c701 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success"; pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream"; pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map"; pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup"; pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate"; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 10dc91362..a11594997 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -1355,6 +1355,7 @@ impl LocalDiskWrapper { self.disk.get_object_path(volume, path) } + #[cfg(unix)] pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result { self.disk.get_object_path_for_io(volume, path) } diff --git a/crates/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs index 8612b6474..7dbbaf14d 100644 --- a/crates/ecstore/src/disk/fs.rs +++ b/crates/ecstore/src/disk/fs.rs @@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result< fs::rename(from, to).await } +#[cfg(any(not(windows), test))] pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<()> { std::fs::rename(from, to) } +#[cfg(any(not(windows), test))] #[tracing::instrument(level = "debug", skip_all)] pub async fn read_file(path: impl AsRef) -> io::Result> { fs::read(path.as_ref()).await diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index a11ed35f0..c24c90724 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -735,7 +735,9 @@ const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_fa /// to replace. Best effort — the rename that follows fails closed — but a /// recurring signal means heal is stuck on that drive. const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed"; +#[cfg(unix)] const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total"; +#[cfg(unix)] const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total"; // io_uring read-backend gray-release observability (rustfs/backlog#1172). #[cfg(target_os = "linux")] @@ -932,10 +934,15 @@ const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_ reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" )] const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false; +#[cfg(any(unix, test))] const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE"; +#[cfg(any(unix, test))] const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false; +#[cfg(any(unix, test))] const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD"; +#[cfg(any(unix, test))] const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy"; +#[cfg(any(unix, test))] const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy"; /// Legacy binary switch for commit-point durability (fsync writes and renames). @@ -951,6 +958,7 @@ const DEFAULT_RUSTFS_DRIVE_SYNC_ENABLE: bool = true; /// See docs/operations/durability-modes.md for the power-loss guarantee matrix. const ENV_RUSTFS_DURABILITY_MODE: &str = "RUSTFS_DURABILITY_MODE"; +#[cfg(any(unix, test))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum LocalReadCopyMethod { MmapCopy, @@ -1359,15 +1367,18 @@ cached_read_env! { cached_read_env! { /// Whether mmap reads should fault the mapping in with `MAP_POPULATE`. + #[cfg(any(unix, test))] fn mmap_populate_enabled() -> bool = rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE); } +#[cfg(any(unix, test))] fn should_populate_mmap_read(length: usize) -> bool { length > 0 && mmap_populate_enabled() } cached_read_env! { + #[cfg(any(unix, test))] fn local_read_copy_method() -> LocalReadCopyMethod = { let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY); match method.as_str() { @@ -2010,7 +2021,7 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se .insert(dst_path.to_string(), Box::new(hook)); } -#[cfg(test)] +#[cfg(all(test, unix))] fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) { INLINE_BEFORE_FILE_SYNC_ADMISSION .lock() @@ -2257,7 +2268,7 @@ fn should_remove_staged_meta_before_commit(_dst_path: &str) -> bool { false } -#[cfg(not(test))] +#[cfg(all(not(test), not(windows)))] fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool { false } @@ -3758,6 +3769,7 @@ struct FdKey { /// The generation fence and explicit mutation invalidation keep the snapshot /// tied to the inode held by `file`, allowing cache hits to avoid a repeated /// metadata syscall without weakening replacement/heal semantics. +#[cfg(unix)] struct FdCacheEntry { /// An independently cloneable descriptor for the immutable shard inode. file: Arc, @@ -5508,6 +5520,7 @@ impl LocalDisk { local_disk_bucket_path(&self.root, bucket) } + #[cfg(any(unix, test))] pub(crate) fn get_object_path_for_io(&self, bucket: &str, key: &str) -> Result { self.io_get_object_path(bucket, key) } @@ -19351,11 +19364,17 @@ mod test { path_resolve_stage: "path", metadata_lookup_stage: "metadata_lookup", metadata_validate_stage: "metadata_validate", + #[cfg(unix)] blocking_wait_stage: "blocking_wait", + #[cfg(unix)] blocking_task_stage: "blocking_task", + #[cfg(unix)] file_open_stage: "file_open", + #[cfg(unix)] mmap_map_stage: "mmap_map", + #[cfg(unix)] mmap_copy_stage: "mmap_copy", + #[cfg(unix)] direct_read_copy_stage: "direct_read_copy", }; diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index f84e1f3c3..ccb950c27 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -17,13 +17,14 @@ #[cfg(all(test, windows))] use super::run_destination_commit_directory_preparation; +#[cfg(any(not(windows), test))] +use super::should_fail_local_inline_rollback_hardlink; use super::{ EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size, remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, - should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit, - skip_access_checks, + should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks, }; #[cfg(test)] use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication}; @@ -88,6 +89,7 @@ fn rollback_inline_metadata_commit_std( Ok(()) } +#[cfg(any(not(windows), test))] pub(super) fn create_local_inline_rollback_backup( dst_file_path: &Path, staging_file_path: &Path, diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index ea80b4a8f..5e6ce9786 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -198,11 +198,17 @@ pub struct MmapCopyStageMetrics { pub(crate) path_resolve_stage: &'static str, pub(crate) metadata_lookup_stage: &'static str, pub(crate) metadata_validate_stage: &'static str, + #[cfg(unix)] pub(crate) blocking_wait_stage: &'static str, + #[cfg(unix)] pub(crate) blocking_task_stage: &'static str, + #[cfg(unix)] pub(crate) file_open_stage: &'static str, + #[cfg(unix)] pub(crate) mmap_map_stage: &'static str, + #[cfg(unix)] pub(crate) mmap_copy_stage: &'static str, + #[cfg(unix)] pub(crate) direct_read_copy_stage: &'static str, } @@ -949,6 +955,7 @@ impl Disk { } } + #[cfg(unix)] pub(crate) fn get_object_path_for_io_if_local( &self, volume: &str, diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index bb57200b2..8b07e8548 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder { static RECORDED: Mutex> = Mutex::new(Vec::new()); static LIMITED: Mutex> = Mutex::new(Vec::new()); static GROUPED: Mutex> = Mutex::new(Vec::new()); + #[cfg(unix)] static BEFORE_LIMITED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); static BEFORE_GROUP_BATCH: std::sync::LazyLock>> = @@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder { contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir) } + #[cfg(unix)] pub(crate) fn record_limited(dir: &Path) { record_path(&LIMITED, dir, "limited fsync dir recorder"); let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned"); @@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder { contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir) } + #[cfg(unix)] pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) { BEFORE_LIMITED .lock() @@ -237,6 +240,7 @@ pub(crate) mod fsync_dir_recorder { .insert(dir.to_path_buf(), kind); } + #[cfg(unix)] pub(crate) fn take_grouped_failure(dir: &Path) -> Option { remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned") } diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 371a93560..d5d5f06de 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -13,12 +13,15 @@ // limitations under the License. use crate::diagnostics::get::{ - GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, - GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, - GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE, + GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE, GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS, GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled, }; +#[cfg(unix)] +use crate::diagnostics::get::{ + GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER, + GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP, +}; #[cfg(feature = "hotpath")] use crate::disk::FileWriter; use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError}; @@ -406,11 +409,17 @@ async fn open_disk_reader( path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE, metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP, metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE, + #[cfg(unix)] blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT, + #[cfg(unix)] blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK, + #[cfg(unix)] file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN, + #[cfg(unix)] mmap_map_stage: GET_STAGE_READER_MMAP_MAP, + #[cfg(unix)] mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER, + #[cfg(unix)] direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY, }); let mmap_result = { diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 59b4d6f4e..09c6dee81 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -56,6 +56,7 @@ use std::collections::HashMap; use std::io::Cursor; +#[cfg(feature = "test-util")] use std::path::Path; use std::sync::{ Arc, @@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt; use tokio::sync::{Mutex, Notify, RwLock}; use uuid::Uuid; +#[cfg(feature = "test-util")] use crate::disk::endpoint::Endpoint; +#[cfg(feature = "test-util")] use crate::disk::format::FormatV3; +#[cfg(feature = "test-util")] use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk}; use crate::services::tier::tier::TierConfigMgr; use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType}; use crate::services::tier::warm_backend::{ TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, }; +#[cfg(feature = "test-util")] use rustfs_filemeta::FileMeta; use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; +#[cfg(feature = "test-util")] use rustfs_utils::path::path_join_buf; /// One-shot barrier before rejected transition cleanup resolves its ECStore. +#[cfg(feature = "test-util")] pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier); +#[cfg(feature = "test-util")] impl TransitionCleanupStoreBarrier { /// Install the barrier for the next rejected transition cleanup. pub fn install() -> Self { @@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier { } /// Default polling cadence used by the `wait_for_*` helpers. +#[cfg(feature = "test-util")] const POLL_INTERVAL: Duration = Duration::from_millis(50); /// A fault to inject into [`MockWarmBackend`] operations. @@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard { } /// One-shot barrier that pauses a mock tier PUT after storing its remote body. +#[cfg(feature = "test-util")] pub struct MockPutBarrier { state: Arc, } +#[cfg(feature = "test-util")] impl MockPutBarrier { /// Wait until the remote body is stored and the PUT is paused before returning. pub async fn wait_until_paused(&self) { @@ -226,6 +237,7 @@ impl MockPutBarrier { } } +#[cfg(feature = "test-util")] impl Drop for MockPutBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -258,10 +270,12 @@ impl Drop for MockGetBarrier { } /// One-shot barrier that pauses and then fails a mock tier DELETE. +#[cfg(feature = "test-util")] pub struct MockRemoveBarrier { state: Arc, } +#[cfg(feature = "test-util")] impl MockRemoveBarrier { /// Wait until DELETE reaches the deterministic failure point. pub async fn wait_until_paused(&self) { @@ -283,6 +297,7 @@ impl MockRemoveBarrier { } } +#[cfg(feature = "test-util")] impl Drop for MockRemoveBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -306,6 +321,7 @@ impl MockWarmBackend { } /// Arm a one-shot pause after the next tier PUT stores its remote body. + #[cfg(feature = "test-util")] pub async fn arm_put_barrier(&self) -> MockPutBarrier { let state = Arc::new(MockPutBarrierState::default()); *self.inner.put_barrier.lock().await = Some(Arc::clone(&state)); @@ -313,6 +329,7 @@ impl MockWarmBackend { } /// Pause and then fail the next DELETE after it reaches the backend. + #[cfg(feature = "test-util")] pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier { let state = Arc::new(MockRemoveBarrierState::default()); let mut barrier = self.inner.remove_barrier.lock().await; @@ -323,6 +340,7 @@ impl MockWarmBackend { /// Arm a one-shot pause before the next tier GET, then return an error /// after the test releases it. + #[cfg(feature = "test-util")] pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier { let state = Arc::new(MockGetBarrierState { fail_after_release: true, @@ -343,6 +361,7 @@ impl MockWarmBackend { // ---- fault injection ------------------------------------------------- /// Replace the entire fault configuration. + #[cfg(feature = "test-util")] pub async fn set_faults(&self, faults: FaultConfig) { *self.inner.faults.lock().await = faults; } @@ -353,6 +372,7 @@ impl MockWarmBackend { } /// Toggle "HTTP 5xx" server errors on every operation. + #[cfg(feature = "test-util")] pub async fn set_server_error(&self, server_error: bool) { self.inner.faults.lock().await.server_error = server_error; } @@ -363,11 +383,13 @@ impl MockWarmBackend { } /// Set (or clear, with `None`) injected latency applied before each op. + #[cfg(feature = "test-util")] pub async fn set_latency(&self, latency: Option) { self.inner.faults.lock().await.latency = latency; } /// Clear all injected faults, restoring healthy behaviour. + #[cfg(feature = "test-util")] pub async fn clear_faults(&self) { *self.inner.faults.lock().await = FaultConfig::default(); } @@ -375,6 +397,7 @@ impl MockWarmBackend { /// Limit how many body bytes a successful mock PUT consumes. `None` drains /// the complete body. This models a backend that incorrectly accepts a /// truncated stream while still returning success. + #[cfg(feature = "test-util")] pub async fn set_put_read_limit(&self, limit: Option) { *self.inner.put_read_limit.lock().await = limit; } @@ -395,12 +418,14 @@ impl MockWarmBackend { } /// Reject non-empty remote versions before transition metadata is committed. + #[cfg(feature = "test-util")] pub fn set_reject_non_empty_remote_versions(&self, reject: bool) { self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release); } /// Reject the next non-empty remote version validation without changing /// subsequent exact-version backend cleanup behavior. + #[cfg(feature = "test-util")] pub fn reject_next_non_empty_remote_version_validation(&self) { self.inner .reject_non_empty_remote_version_validations @@ -438,6 +463,7 @@ impl MockWarmBackend { } /// Clear the operation log without touching stored objects or faults. + #[cfg(feature = "test-util")] pub async fn clear_op_log(&self) { self.inner.op_log.lock().await.clear(); } @@ -459,11 +485,13 @@ impl MockWarmBackend { } /// Return the exact object/version pairs produced by successful tier PUTs. + #[cfg(feature = "test-util")] pub async fn put_versions(&self) -> Vec<(String, String)> { self.inner.put_versions.lock().await.clone() } /// Return the exact object/version pairs passed to successful tier removes. + #[cfg(feature = "test-util")] pub async fn remove_versions(&self) -> Vec<(String, String)> { self.inner.remove_versions.lock().await.clone() } @@ -475,6 +503,7 @@ impl MockWarmBackend { /// Number of `get` calls recorded — useful to assert restore reads hit the /// local copy rather than the remote tier. + #[cfg(feature = "test-util")] pub async fn get_count(&self) -> usize { self.inner .op_log @@ -486,6 +515,7 @@ impl MockWarmBackend { } /// Number of `put` calls recorded. + #[cfg(feature = "test-util")] pub async fn put_count(&self) -> usize { self.inner .op_log @@ -499,6 +529,7 @@ impl MockWarmBackend { // ---- storage inspection --------------------------------------------- /// Whether the backend currently stores `object`. + #[cfg(feature = "test-util")] pub async fn contains(&self, object: &str) -> bool { self.inner.objects.lock().await.contains_key(object) } @@ -509,11 +540,13 @@ impl MockWarmBackend { } /// A clone of the stored object, if present. + #[cfg(feature = "test-util")] pub async fn stored(&self, object: &str) -> Option { self.inner.objects.lock().await.get(object).cloned() } /// A clone of the raw bytes stored for `object`, if present. + #[cfg(feature = "test-util")] pub async fn bytes(&self, object: &str) -> Option> { self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone()) } @@ -538,6 +571,7 @@ impl MockWarmBackend { /// Poll until `object` is absent from the backend, or `timeout` elapses. /// Returns `true` if the object disappeared within the budget. + #[cfg(feature = "test-util")] pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -553,6 +587,7 @@ impl MockWarmBackend { /// Poll until the backend holds exactly `expected` objects, or `timeout` /// elapses. Returns `true` if the count was reached within the budget. + #[cfg(feature = "test-util")] pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc>, tie /// The transition-state tuple read from an on-disk `xl.meta`, plus the object's /// free-version count. #[derive(Clone, Debug, PartialEq, Eq)] +#[cfg(feature = "test-util")] pub struct TransitionMeta { /// `transition_status` (e.g. `"complete"`), empty when not transitioned. pub status: String, @@ -860,6 +896,7 @@ pub struct TransitionMeta { pub free_version_count: usize, } +#[cfg(feature = "test-util")] async fn open_disk(disk_path: &Path) -> Option { // `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees // with the position recorded in the disk's own format.json, so derive the @@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option { /// The free-version metadata removal lands asynchronously after the remote /// object disappears, so callers typically poll via /// [`wait_for_free_version_absence`] instead of asserting a single read. +#[cfg(feature = "test-util")] pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize { let Some(disk) = open_disk(disk_path).await else { return 0; @@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> /// fields are taken from the newest version that carries a transition record; /// if no version is transitioned, they are taken from the current version (and /// will be empty). +#[cfg(feature = "test-util")] pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option { let disk = open_disk(disk_path).await?; let data = disk @@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) /// disk is missing the object or disagrees — this is the shard-consistency /// check required by ilm-6 (the `(status, tier, remote key, remote version id)` /// four-tuple plus free-version count must match across all erasure shards). +#[cfg(feature = "test-util")] pub async fn assert_transition_meta_consistent>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta { assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk"); @@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent>(disk_paths: &[P], /// Poll until `object` retains no free versions on `disk_path`, or `timeout` /// elapses. Returns `true` if the free versions drained within the budget. +#[cfg(feature = "test-util")] pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -1040,6 +1081,44 @@ mod tests { ); } + #[tokio::test] + async fn mock_metadata_survives_put_and_external_delete_is_distinct() { + let backend = MockWarmBackend::new(); + let metadata = HashMap::from([ + ("content-type".to_string(), "text/plain".to_string()), + ("project".to_string(), "archive".to_string()), + ]); + let version = backend + .put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone()) + .await + .expect("mock PUT should preserve remote metadata"); + assert_eq!(backend.metadata("object").await, Some(metadata)); + assert_eq!( + backend + .probe_transition_candidate_state("object") + .await + .expect("probe stored object"), + TransitionCandidateProbe::VersionedPresent(version) + ); + + backend.external_remove("object").await; + assert_eq!(backend.metadata("object").await, None); + assert_eq!( + backend + .probe_transition_candidate_state("object") + .await + .expect("probe removed object"), + TransitionCandidateProbe::Missing + ); + let operations = backend.op_log().await; + assert!( + operations + .iter() + .any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object")) + ); + assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. }))); + } + #[tokio::test] async fn mock_probe_preserves_fault_fail_closed_behavior() { let backend = MockWarmBackend::new(); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 04ff49743..4dc17ad34 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -111,10 +111,11 @@ use crate::disk::{ use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; use crate::io_support::bitrot::{ - BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params, - create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle, - object_mmap_read_max_length, + BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics, + create_deferred_bitrot_reader_with_stripe_handle, }; +#[cfg(unix)] +use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length}; use crate::set_disk::runtime_sources; use crate::set_disk::shard_source::ShardReadCost; use crate::storage_api_contracts::object::ObjectOperations; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 2c076efea..179c4fd38 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation; pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; #[cfg(test)] pub(crate) use ops::object::DeleteObjectCommitBarrier; -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; #[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 3e09515d2..4fcb450bd 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably( } async fn transition_cleanup_store(ctx: &Arc) -> Option> { - #[cfg(any(test, feature = "test-util"))] + #[cfg(feature = "test-util")] pause_transition_cleanup_store().await; transition_object_store(ctx).await @@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup( } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] #[derive(Default)] struct TransitionCleanupStoreBarrierState { arrived: tokio::sync::Notify, release: tokio::sync::Notify, } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] /// One-shot test barrier placed before transition cleanup resolves its ECStore. pub(crate) struct TransitionCleanupStoreBarrier { state: Arc, } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] impl TransitionCleanupStoreBarrier { /// Install the process-local barrier for the next cleanup-store resolution. pub(crate) fn install() -> Self { @@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier { } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] impl Drop for TransitionCleanupStoreBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier { } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] async fn pause_transition_cleanup_store() { let barrier = TRANSITION_CLEANUP_STORE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] struct TransitionUploadedCommitBarrierState { bucket: String, object: String, @@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionUploadedCommitBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl TransitionUploadedCommitBarrier { pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(TransitionUploadedCommitBarrierState { @@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for TransitionUploadedCommitBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) { let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } upload_cleanup.update_cleanup_transaction(&transaction); - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pause_after_transition_uploaded_persisted(bucket, object).await; let commit_opts = opts.as_commit_opts(); @@ -12672,6 +12672,65 @@ mod metadata_mutation_generation_tests { set_disks.invalidate_get_object_metadata_cache(bucket, object).await; } + #[tokio::test] + #[serial_test::serial(metadata_cache_invalidation_probe)] + async fn segment_observation_equal_size_mutations_retire_metadata_generation() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "segment-observation-bucket"; + let object = "hot/object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("create segment fixture bucket"); + } + let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await; + let probe = MetadataCacheInvalidationProbe::install(bucket, object); + let mut replacement = PutObjReader::from_vec(b"after!".to_vec()); + set_disks + .put_object(bucket, object, &mut replacement, &ObjectOptions::default()) + .await + .expect("commit same-length replacement with normal owner locking"); + assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation"); + assert_retired(&set_disks, &old_key).await; + drop(probe); + let after = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read replacement metadata"); + assert_eq!(before.size, after.size); + assert_ne!(before.etag, after.etag, "equal size is not equal content"); + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read replacement body through the owner"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("drain replacement body"); + assert_eq!(body, b"after!"); + drop(reader); + + let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await; + let probe = MetadataCacheInvalidationProbe::install(bucket, object); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])), + ..Default::default() + }, + ) + .await + .expect("commit metadata-only mutation with normal owner locking"); + assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences"); + assert_retired(&set_disks, &old_key).await; + let after = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read committed metadata-only mutation"); + assert_eq!(before.size, after.size); + assert_eq!(before.etag, after.etag); + assert!(!before.user_defined.contains_key("x-amz-meta-segment")); + assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed")); + } + #[tokio::test] #[serial_test::serial(metadata_cache_invalidation_probe)] async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() { diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 908572998..469a5e365 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation( } impl ECStore { - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) { data_movement_multipart_discovery_counts() .lock() @@ -246,7 +246,7 @@ impl ECStore { .insert(self.id, 0); } - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize { data_movement_multipart_discovery_counts() .lock() diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index 336ba00ad..7fa8d7717 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -54,6 +54,15 @@ pub enum Error { #[error("Heal task execution failed: {message}")] TaskExecutionFailed { message: String }, + /// The current page already exhausted its local retry budget. Retrying + /// the enclosing bucket would replay pages whose results were counted. + #[error("Heal listing failed for bucket {bucket}: {source}")] + HealListingFailed { + bucket: String, + #[source] + source: Box, + }, + #[error("Invalid heal type: {heal_type}")] InvalidHealType { heal_type: String }, diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 33ac33c38..580b01246 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -447,6 +447,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "running".to_string(), None, @@ -463,6 +464,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "running".to_string(), Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")), @@ -479,6 +481,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "finished".to_string(), None, @@ -495,6 +498,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some("heal task cancelled".to_string()), @@ -511,6 +515,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some("heal task timed out".to_string()), @@ -527,6 +532,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some(error), diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 971590fdb..c90a18729 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::heal::{ + outcome::HealTaskOutcome, progress::{HealProgress, HealStatistics}, resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils}, storage::HealStorageAPI, @@ -185,6 +186,7 @@ fn record_displaced_terminal( request: &HealRequest, ) -> Arc { let terminal = Arc::new(CompletedHealStatus { + outcome: None, progress: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), @@ -268,6 +270,7 @@ async fn publish_completed_heal( #[derive(Debug, Clone)] pub struct HealTaskReport { + pub outcome: Option>, pub status: HealTaskStatus, pub result_items: Vec, pub result_items_truncated: bool, @@ -285,6 +288,7 @@ async fn active_task_report(task: &HealTask, since: Option) -> HealTaskRepo let window = task.get_result_items_since(since).await; HealTaskReport { status: task.get_status().await, + outcome: Some(Arc::new(task.get_outcome().await)), result_items: window.items, // The legacy flag stays set once anything was evicted; a lagging // incremental cursor additionally marks this response truncated so @@ -298,6 +302,7 @@ async fn active_task_report(task: &HealTask, since: Option) -> HealTaskRepo fn empty_task_report(status: HealTaskStatus) -> HealTaskReport { HealTaskReport { + outcome: None, status, result_items: Vec::new(), result_items_truncated: false, @@ -325,6 +330,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> }; HealTaskReport { status: completed.status.clone(), + outcome: completed.outcome.clone(), result_items, result_items_truncated: completed.result_items_truncated || lagged, progress: completed.progress.clone(), diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index aceabe42a..a47cd43e2 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -83,6 +83,7 @@ pub(super) struct CompletedHealStatus { pub(super) heal_type: HealType, pub(super) status: HealTaskStatus, pub(super) progress: Option, + pub(super) outcome: Option>, pub(super) retained_bytes: std::sync::OnceLock, pub(super) result_items_truncated: bool, pub(super) completed_at: SystemTime, @@ -105,6 +106,7 @@ impl CompletedHealStatus { fn measure_retained_bytes(&self) -> usize { let mut bytes = size_of::(); let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes())); match &self.heal_type { HealType::Cluster => {} HealType::Bucket { bucket } => add(bucket.capacity()), @@ -209,6 +211,7 @@ impl CompletedHealStatus { heal_type: task.heal_type.clone(), status, progress: Some(task.get_progress().await), + outcome: Some(Arc::new(task.get_outcome().await)), retained_bytes: std::sync::OnceLock::new(), result_items_truncated: task.result_items_truncated(), completed_at: SystemTime::now(), diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index feb0c1231..704a7cbeb 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -298,6 +298,7 @@ impl HealManager { if cancelled_completion { completed_status = HealTaskStatus::Cancelled; completed_status_entry.status = HealTaskStatus::Cancelled; + completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await)); } let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); let successful_completion = matches!(completed_status, HealTaskStatus::Completed); diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index aa1c1293f..d8b3e3b16 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -103,6 +103,7 @@ struct MockStorage; fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus { CompletedHealStatus { + outcome: None, heal_type: HealType::Cluster, status: HealTaskStatus::Completed, progress: Some(HealProgress { @@ -287,6 +288,59 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat } } +#[tokio::test] +async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() { + use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome}; + use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook}; + let bucket = "canonical-outcome-cancel-before-finish"; + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let alias = duplicate.id.clone(); + let retention_hook = Arc::new(CompletedRetentionHook::default()); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.to_string(), retention_hook.clone()); + hooks.insert(task_id.clone(), retention_hook.clone()); + } + let finish_hook = Arc::new(OutcomeFinishTestHook { + task_id: task_id.clone(), + reached: Notify::new(), + release: Notify::new(), + }); + *OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone()); + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified()) + .await + .expect("storage started"); + retention_hook.execute.notify_one(); + tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified()) + .await + .expect("storage returned before outcome finalization"); + manager.cancel_task(&alias).await.expect("cancel wins publication"); + finish_hook.release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified()) + .await + .expect("scheduler completes cancelled handoff"); + for token in [&task_id, &alias] { + let report = manager.get_task_report(token).await.expect("cancelled token retained"); + assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!( + report.outcome.as_ref().expect("frozen outcome").execution, + HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) + ); + } + retention_hook.finish.notify_one(); + *OUTCOME_FINISH_TEST_HOOK.lock().await = None; + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != bucket && key != &task_id); +} + #[tokio::test] async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { let bucket = "completed-retention-retry-cancel"; @@ -325,6 +379,10 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { for token in [&task_id, &alias] { let report = manager.get_task_report(token).await.expect("cancelled token retained"); assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!( + report.outcome.as_ref().expect("cancelled outcome retained").execution, + crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled) + ); assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1); } assert!(!manager.retrying_heals.lock().await.contains_key(&task_id)); @@ -391,6 +449,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han .expect("scheduler archives terminal"); assert!(!manager.active_heals.lock().await.contains_key(&task_id)); let expected = task.get_progress().await; + let expected_outcome = task.get_outcome().await; for token in [&task_id, &alias] { assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected); let report = manager @@ -398,6 +457,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han .await .expect("terminal token remains queryable at handoff"); assert_eq!(report.progress.as_ref(), Some(&expected)); + assert_eq!(report.outcome.as_deref(), Some(&expected_outcome)); assert!(report.result_items.is_empty()); match outcome { "success" => assert_eq!(report.status, HealTaskStatus::Completed), @@ -1976,6 +2036,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> task_id, Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type, status: HealTaskStatus::Retrying { @@ -2700,6 +2761,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { task_id.clone(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Retrying { @@ -2737,6 +2799,7 @@ async fn test_get_task_status_reads_recent_completed_status() { "completed-token".to_string(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Bucket { bucket: "bucket".to_string(), @@ -2768,6 +2831,7 @@ async fn test_get_task_report_for_path_reads_completed_items() { "completed-token".to_string(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Object { bucket: "bucket".to_string(), diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index c918bea49..5f17c8cd8 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -16,6 +16,7 @@ pub mod channel; pub mod erasure_healer; pub mod manager; pub mod mrf_queue; +pub mod outcome; pub mod progress; pub(crate) mod replacement_readiness; pub mod resume; diff --git a/crates/heal/src/heal/mrf_queue/snapshot.rs b/crates/heal/src/heal/mrf_queue/snapshot.rs index e8d51ed9e..d39e0fafa 100644 --- a/crates/heal/src/heal/mrf_queue/snapshot.rs +++ b/crates/heal/src/heal/mrf_queue/snapshot.rs @@ -33,6 +33,9 @@ use std::collections::HashMap; use tokio::io::AsyncReadExt; use uuid::Uuid; +/// Explicit pending migration; never activates the production writer or GC. +pub mod migration; + // Root-level control files avoid requiring a new directory before the first // atomic commit. They remain inside the storage owner's metadata volume. const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"]; @@ -66,6 +69,23 @@ struct Manifest { } impl Manifest { + fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result, SnapshotError> { + let mut bytes = Vec::with_capacity(MANIFEST_LEN); + bytes.extend_from_slice(MAGIC); + bytes.push(VERSION); + bytes.extend_from_slice(owner.as_bytes()); + bytes.extend_from_slice(&sequence.to_le_bytes()); + bytes.extend_from_slice( + &u64::try_from(payload.len()) + .map_err(|_| SnapshotError::TooLarge)? + .to_le_bytes(), + ); + bytes.extend_from_slice(&Sha256::digest(payload)); + bytes.extend_from_slice(&Sha256::digest(&bytes)); + Self::decode(&bytes, payload.len())?; + Ok(bytes) + } + fn decode(bytes: &[u8], limit: usize) -> Result { if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC { return Err(SnapshotError::Corrupt); diff --git a/crates/heal/src/heal/mrf_queue/snapshot/migration.rs b/crates/heal/src/heal/mrf_queue/snapshot/migration.rs new file mode 100644 index 000000000..97dedaa59 --- /dev/null +++ b/crates/heal/src/heal/mrf_queue/snapshot/migration.rs @@ -0,0 +1,1450 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +//! Pending, owner-local legacy import. These paths are deliberately invisible +//! to the active snapshot reader and legacy consumer. Source revalidation is +//! not a writer freeze: no result here grants activation or reclamation rights. +//! All source bytes and inherited responsibilities survive admission/replay. + +use super::{MANIFEST_LEN, Manifest, SnapshotError, read_bounded}; +use crate::heal::RUSTFS_META_BUCKET; +use crate::heal::mrf_queue::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_one}; +use crate::heal::storage_api::owner::{ + EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskStore, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use tokio::io::AsyncReadExt; +use uuid::Uuid; + +const PAYLOADS: [&str; 2] = [".heal-mrf-import-pending.0.bin", ".heal-mrf-import-pending.1.bin"]; +const COMMITS: [&str; 2] = [".heal-mrf-import-commit.0.bin", ".heal-mrf-import-commit.1.bin"]; +const MAX_DISKS: usize = 64; +const CLAIM: &str = ".heal-mrf-import-claim.bin"; + +/// Limits apply to the complete encoded candidate and all distinct raw records, +/// including inherited sources. Exceeding either preserves previous anchors. +#[derive(Clone, Copy, Debug)] +pub struct MigrationLimits { + pub max_bytes: usize, + pub max_records: usize, + pub max_sources: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum MigrationError { + #[error(transparent)] + Snapshot(#[from] SnapshotError), + #[error("MRF migration requires every configured, formatted local disk")] + CoverageGap, + #[error("MRF migration source changed; all recovery anchors are retained")] + SourceChanged, + #[error("MRF migration candidate is invalid")] + Invalid, + #[error("MRF migration has no responsibility evidence")] + Empty, + #[error("MRF migration conditional publication conflicted")] + Conflict, + #[error("MRF migration staging is claimed; interrupted claims require separately fenced recovery")] + Claimed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +enum LegacyPath { + Scoped, + Mirror, +} + +impl LegacyPath { + fn path(self) -> &'static str { + match self { + Self::Scoped => MRF_SCOPED_JOURNAL_PATH, + Self::Mirror => MRF_JOURNAL_PATH, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Source { + disk_id: Uuid, + path: LegacyPath, + digest: [u8; 32], + // None proves an observed absent path, distinct from a present empty file. + bytes: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PendingMigration { + version: u8, + sources: Vec, + inherited: Vec, +} + +impl PendingMigration { + /// Raw, complete records. No scope/version normalization, attempts pruning, + /// incarnation inference or task-success interpretation is performed. + pub fn replay_records(&self, limits: MigrationLimits) -> Result>, MigrationError> { + self.validate_limits(limits)?; + let mut records = BTreeSet::new(); + for source in self.sources.iter().chain(&self.inherited) { + if source.disk_id.is_nil() { + return Err(MigrationError::Invalid); + } + let bytes = source.bytes.as_deref().unwrap_or_default(); + if <[u8; 32]>::from(Sha256::digest(bytes)) != source.digest { + return Err(MigrationError::Invalid); + } + let mut offset = 0; + while offset < bytes.len() { + let (_, consumed) = decode_one(&bytes[offset..]).ok_or(MigrationError::Invalid)?; + let end = offset.checked_add(consumed).ok_or(MigrationError::Invalid)?; + records.insert(bytes[offset..end].to_vec()); + if records.len() > limits.max_records { + return Err(SnapshotError::TooLarge.into()); + } + offset = end; + } + } + Ok(records.into_iter().collect()) + } + + fn validate_limits(&self, limits: MigrationLimits) -> Result<(), MigrationError> { + if self.version != 1 { + return Err(SnapshotError::Unsupported.into()); + } + if self.sources.is_empty() || self.sources.len() > MAX_DISKS * 2 { + return Err(MigrationError::Invalid); + } + if self + .sources + .len() + .checked_add(self.inherited.len()) + .is_none_or(|count| count > limits.max_sources) + { + return Err(SnapshotError::TooLarge.into()); + } + // Bound the raw input before allocating the JSON representation. + let total = self + .sources + .iter() + .chain(&self.inherited) + .try_fold(0usize, |total, source| { + total + .checked_add(source.bytes.as_ref().map_or(0, Vec::len)) + .and_then(|n| n.checked_add(128)) + }) + .ok_or(SnapshotError::TooLarge)?; + if total > limits.max_bytes { + return Err(SnapshotError::TooLarge.into()); + } + Ok(()) + } + + fn encode(&self, limits: MigrationLimits) -> Result, MigrationError> { + self.replay_records(limits)?; + let bytes = serde_json::to_vec(self).map_err(|_| MigrationError::Invalid)?; + if bytes.len() > limits.max_bytes { + return Err(SnapshotError::TooLarge.into()); + } + Ok(bytes) + } + + async fn revalidate(&self, disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result<(), MigrationError> { + let current = capture(disks, limits).await?; + if current.sources != self.sources { + return Err(MigrationError::SourceChanged); + } + Ok(()) + } +} + +async fn configured_disks(disks: &[Option]) -> Result, MigrationError> { + if disks.is_empty() || disks.len() > MAX_DISKS { + return Err(MigrationError::CoverageGap); + } + let mut ordered = std::collections::BTreeMap::new(); + for disk in disks { + let disk = disk.as_ref().ok_or(MigrationError::CoverageGap)?; + if !EcstoreDiskAPI::is_local(disk.as_ref()) { + return Err(MigrationError::CoverageGap); + } + let id = EcstoreDiskAPI::get_disk_id(disk.as_ref()) + .await + .map_err(SnapshotError::Disk)? + .filter(|id| !id.is_nil()) + .ok_or(MigrationError::CoverageGap)?; + if ordered.insert(id, disk.clone()).is_some() { + return Err(MigrationError::CoverageGap); + } + } + Ok(ordered.into_values().collect()) +} + +async fn capture(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result { + let mut sources = Vec::with_capacity(disks.len() * 2); + let mut identities = BTreeSet::new(); + let mut remaining = limits.max_bytes; + for disk in disks { + let id = EcstoreDiskAPI::get_disk_id(disk.as_ref()) + .await + .map_err(SnapshotError::Disk)? + .filter(|id| !id.is_nil()) + .ok_or(MigrationError::CoverageGap)?; + if !identities.insert(id) { + return Err(MigrationError::CoverageGap); + } + for path in [LegacyPath::Scoped, LegacyPath::Mirror] { + // A missing metadata volume is a coverage gap, not an absent journal. + let bytes = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path.path()).await { + Ok(reader) => { + let maximum = u64::try_from(remaining.checked_add(1).ok_or(SnapshotError::TooLarge)?) + .map_err(|_| SnapshotError::TooLarge)?; + let mut bytes = Vec::new(); + reader + .take(maximum) + .read_to_end(&mut bytes) + .await + .map_err(SnapshotError::Read)?; + if bytes.len() > remaining { + return Err(SnapshotError::TooLarge.into()); + } + Some(bytes) + } + Err(EcstoreDiskError::FileNotFound) => None, + Err(error) => return Err(SnapshotError::Disk(error).into()), + }; + remaining = remaining + .checked_sub(bytes.as_ref().map_or(0, Vec::len)) + .ok_or(SnapshotError::TooLarge)?; + let digest = Sha256::digest(bytes.as_deref().unwrap_or_default()).into(); + sources.push(Source { + disk_id: id, + path, + digest, + bytes, + }); + } + } + sources.sort_by_key(|source| (source.disk_id, matches!(source.path, LegacyPath::Mirror))); + let candidate = PendingMigration { + version: 1, + sources, + inherited: Vec::new(), + }; + candidate.encode(limits)?; + Ok(candidate) +} + +/// Inspect every configured source without merging v1 mirrors into a claimed +/// latest snapshot. A complete subset/superset is retained as pending evidence. +pub async fn capture_legacy_migration( + disks: &[Option], + limits: MigrationLimits, +) -> Result { + capture(&configured_disks(disks).await?, limits).await +} + +struct Staged { + manifest: Manifest, + candidate: PendingMigration, + slot: usize, +} + +type PayloadIdentity = (usize, [u8; 32]); + +struct StagingLineage { + latest: Option, + // Each collection has at most two identities per configured disk. + committed_payloads: BTreeSet, + orphaned_payloads: Vec, +} + +fn payload_identity(payload: &[u8]) -> PayloadIdentity { + (payload.len(), Sha256::digest(payload).into()) +} + +impl StagingLineage { + fn validate_orphans(&self, retry_payload: Option<&[u8]>) -> Result<(), MigrationError> { + let retry_identity = retry_payload.map(payload_identity); + if self + .orphaned_payloads + .iter() + .any(|identity| Some(*identity) != retry_identity && !self.committed_payloads.contains(identity)) + { + return Err(MigrationError::Conflict); + } + Ok(()) + } +} + +async fn read_staging_lineage(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result { + let mut selected: Option = None; + let mut identities = std::collections::BTreeMap::new(); + let mut orphaned_payloads = Vec::new(); + let mut committed_payloads = BTreeSet::new(); + let mut mismatched_manifests = Vec::new(); + for disk in disks { + for slot in 0..2 { + let result = async { + let payload = read_bounded(disk, PAYLOADS[slot], limits.max_bytes).await?; + let Some(bytes) = read_bounded(disk, COMMITS[slot], MANIFEST_LEN).await? else { + if let Some(payload) = payload.as_deref() { + orphaned_payloads.push(payload_identity(payload)); + } + return Ok(None); + }; + let manifest = Manifest::decode(&bytes, limits.max_bytes)?; + let identity = (manifest.owner, manifest.payload_digest); + if identities + .insert(manifest.sequence, identity) + .is_some_and(|old| old != identity) + { + return Err(MigrationError::Conflict); + } + let payload = payload.ok_or(MigrationError::Invalid)?; + if payload.len() != manifest.payload_len || payload_identity(&payload).1 != manifest.payload_digest { + // A reused slot can hold the next retry payload while the + // old manifest still names the previous generation. + orphaned_payloads.push(payload_identity(&payload)); + mismatched_manifests.push(manifest); + return Ok(None); + } + let candidate: PendingMigration = serde_json::from_slice(&payload).map_err(|_| MigrationError::Invalid)?; + if candidate.encode(limits)? != payload || candidate.replay_records(limits)?.is_empty() { + return Err(MigrationError::Invalid); + } + Ok(Some(Staged { + manifest, + candidate, + slot, + })) + } + .await; + match result { + Ok(Some(next)) => { + committed_payloads.insert((next.manifest.payload_len, next.manifest.payload_digest)); + if selected + .as_ref() + .is_none_or(|old| old.manifest.sequence < next.manifest.sequence) + { + selected = Some(next); + } + } + Ok(None) => {} + Err(error) => return Err(error), + } + } + } + // Only a validated successor can supersede an unmatched older manifest. + // A newer or unordered manifest may still name responsibilities absent from + // the remaining payload, even if that payload matches another valid slot. + if mismatched_manifests.iter().any(|manifest| { + selected + .as_ref() + .is_none_or(|latest| latest.manifest.owner != manifest.owner || latest.manifest.sequence <= manifest.sequence) + }) { + return Err(MigrationError::Invalid); + } + Ok(StagingLineage { + latest: selected, + committed_payloads, + orphaned_payloads, + }) +} + +async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8], limit: usize) -> Result<(), MigrationError> { + let expected = read_bounded(disk, path, limit).await?.map(EcstoreDiskBytes::from); + let result = EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(EcstoreDiskBytes::copy_from_slice(bytes)), + ) + .await + .map_err(SnapshotError::Disk)?; + if result != EcstoreConditionalFileUpdate::Updated { + return Err(MigrationError::Conflict); + } + Ok(()) +} + +/// Persist an explicitly requested pending import using the storage owner's CAS +/// (and its configured metadata durability). This does not freeze legacy ingress +/// or grant a durable-acceptance/GC receipt. Activation requires W14/W21 evidence. +pub async fn stage_legacy_migration( + disks: &[Option], + candidate: &PendingMigration, + owner: Uuid, + limits: MigrationLimits, +) -> Result { + let disks = configured_disks(disks).await?; + // Claim every configured disk in identity order. A crash/cancellation leaves + // claims intact; a new process cannot guess that the old writer is fenced. + let claim = EcstoreDiskBytes::copy_from_slice(Uuid::new_v4().as_bytes()); + let mut claimed = Vec::new(); + let result = async { + for disk in &disks { + match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, None, Some(claim.clone())) + .await + .map_err(SnapshotError::Disk)? + { + EcstoreConditionalFileUpdate::Updated => claimed.push(disk.clone()), + _ => return Err(MigrationError::Claimed), + } + } + stage_claimed(&disks, candidate, owner, limits).await + } + .await; + let mut release_error = None; + for disk in claimed { + let release = async { + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::Release).await?; + let released = + EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, Some(claim.clone()), None) + .await + .map_err(SnapshotError::Disk)?; + if released != EcstoreConditionalFileUpdate::Updated { + return Err(MigrationError::Claimed); + } + Ok(()) + } + .await; + if let Err(error) = release + && release_error.is_none() + { + release_error = Some(error); + } + } + if let Some(error) = release_error { + return Err(error); + } + result +} + +async fn stage_claimed( + disks: &[EcstoreDiskStore], + candidate: &PendingMigration, + owner: Uuid, + limits: MigrationLimits, +) -> Result { + candidate.revalidate(disks, limits).await?; + candidate.validate_limits(limits)?; + let lineage = read_staging_lineage(disks, limits).await?; + let previous = lineage.latest.as_ref(); + if previous.is_some_and(|old| old.manifest.owner != owner) { + return Err(MigrationError::Conflict); + } + let mut candidate = candidate.clone(); + let key = |source: &Source| (source.disk_id, source.path, source.digest, source.bytes.is_some()); + let mut source_index = std::collections::HashMap::new(); + for (index, source) in candidate.sources.iter().chain(&candidate.inherited).enumerate() { + if source_index.insert(key(source), index).is_some() { + return Err(MigrationError::Invalid); + } + } + if let Some(old) = previous { + for source in old.candidate.sources.iter().chain(&old.candidate.inherited) { + if let Some(index) = source_index.get(&key(source)).copied() { + let existing = if index < candidate.sources.len() { + &candidate.sources[index] + } else { + &candidate.inherited[index - candidate.sources.len()] + }; + if existing != source { + return Err(MigrationError::Conflict); + } + } else { + if source_index.len() >= limits.max_sources { + return Err(SnapshotError::TooLarge.into()); + } + source_index.insert(key(source), candidate.sources.len() + candidate.inherited.len()); + candidate.inherited.push(source.clone()); + } + } + } + if candidate.replay_records(limits)?.is_empty() { + return Err(MigrationError::Empty); + } + let payload = candidate.encode(limits)?; + // Exact retry comparison must include all inherited responsibilities. + // Validating only the freshly captured sources would reject our own + // interrupted successor payload before it can be completed. + lineage.validate_orphans(Some(&payload))?; + let digest: [u8; 32] = Sha256::digest(&payload).into(); + let (sequence, slot) = previous.map_or((1, 0), |old| { + if old.manifest.payload_digest == digest { + (old.manifest.sequence, old.slot) + } else { + (old.manifest.sequence.saturating_add(1), 1 - old.slot) + } + }); + let manifest = Manifest::encode(owner, sequence, &payload)?; + for disk in disks { + install(disk, PAYLOADS[slot], &payload, limits.max_bytes).await?; + } + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterPayload).await?; + candidate.revalidate(disks, limits).await?; + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::BeforeManifest).await?; + for disk in disks { + install(disk, COMMITS[slot], &manifest, MANIFEST_LEN).await?; + } + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterManifest).await?; + let readback = read_staging_lineage(disks, limits).await?; + readback.validate_orphans(None)?; + let recovered = readback.latest.ok_or(MigrationError::Invalid)?; + if recovered.manifest.sequence != sequence || recovered.manifest.payload_digest != digest { + return Err(MigrationError::Conflict); + } + recovered.candidate.revalidate(disks, limits).await?; + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterReadback).await?; + Ok(sequence) +} + +/// Reload pending obligations after process restart. Manager admission never +/// removes them. Missing/changed sources block migration, preserving all files. +pub async fn recover_pending_migration( + disks: &[Option], + limits: MigrationLimits, +) -> Result, MigrationError> { + let disks = configured_disks(disks).await?; + let lineage = read_staging_lineage(&disks, limits).await?; + lineage.validate_orphans(None)?; + let Some(staged) = lineage.latest else { + return Ok(None); + }; + staged.candidate.revalidate(&disks, limits).await?; + Ok(Some(staged.candidate)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::heal::mrf_queue::encode_intent; + use crate::heal::{DiskOption, Endpoint, new_disk}; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope}; + use std::sync::Arc; + use tempfile::TempDir; + + const LIMITS: MigrationLimits = MigrationLimits { + max_bytes: 64 * 1024, + max_records: 100, + max_sources: 64, + }; + + #[derive(Clone, Copy, PartialEq, Eq)] + pub(super) enum Boundary { + AfterPayload, + BeforeManifest, + AfterManifest, + AfterReadback, + Release, + } + + static INTERRUPTIONS: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + + type SourceChange = (EcstoreDiskStore, Vec); + type SourceChangeMap = std::collections::BTreeMap; + + static SOURCE_CHANGES: std::sync::LazyLock> = std::sync::LazyLock::new(Default::default); + + pub(super) async fn interrupt_at(owner: Uuid, boundary: Boundary) -> Result<(), MigrationError> { + if boundary == Boundary::AfterPayload { + let change = SOURCE_CHANGES.lock().expect("source fault map").remove(&owner); + if let Some((disk, bytes)) = change { + source(&disk, &bytes).await; + } + } + let mut interruptions = INTERRUPTIONS.lock().expect("fault map"); + if interruptions.get(&owner) == Some(&boundary) { + interruptions.remove(&owner); + return Err(SnapshotError::Read(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "injected migration boundary failure", + )) + .into()); + } + Ok(()) + } + + fn record(object: &str, kind: MrfKind, scope: Option) -> Vec { + let mut bytes = Vec::new(); + assert!(encode_intent( + &MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from(object), + version_id: None, + kind, + scope, + lease: None, + enqueued_at_ms: 1, + attempts: 255, + }, + &mut bytes + )); + bytes + } + + async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore { + let path = root.path().join(name); + std::fs::create_dir_all(&path).expect("create test disk"); + let mut endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("disk endpoint"); + endpoint.set_idx = 0; + endpoint.disk_idx = 0; + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open disk"); + let created = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await; + assert!( + matches!(created, Ok(()) | Err(EcstoreDiskError::VolumeExists)), + "metadata volume: {created:?}" + ); + let id = Uuid::new_v4(); + let format = serde_json::json!({ + "version": "1", "format": "xl-single", "id": Uuid::new_v4(), + "xl": { "version": "3", "this": id, "sets": [[id]], "distributionAlgo": "SIPMOD+PARITY" } + }); + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + "format.json", + serde_json::to_vec(&format).expect("format").into(), + ) + .await + .expect("format disk"); + assert_eq!(EcstoreDiskAPI::get_disk_id(disk.as_ref()).await.expect("formatted identity"), Some(id)); + disk + } + + async fn source(disk: &EcstoreDiskStore, bytes: &[u8]) { + // Legacy source paths predate the root-level COW control files. + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + EcstoreDiskBytes::copy_from_slice(bytes), + ) + .await + .expect("write legacy source"); + } + + #[tokio::test] + async fn migration_subset_union_preserves_sources_across_reopen() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let a = record("a", MrfKind::PartialWrite, None); + let b = record( + "b", + MrfKind::DecodeFailure, + Some(MrfScope { + pool_index: 0, + set_index: 1, + }), + ); + source(&first, &a).await; + source(&second, &[a.clone(), b.clone()].concat()).await; + let disks = [Some(first.clone()), Some(second.clone())]; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture both complete sources"); + let reverse = capture_legacy_migration(&[Some(second.clone()), Some(first.clone())], LIMITS) + .await + .expect("reverse disk order"); + assert_eq!( + candidate.encode(LIMITS).expect("candidate"), + reverse.encode(LIMITS).expect("reversed candidate") + ); + assert_eq!(candidate.replay_records(LIMITS).expect("raw records").len(), 2); + let owner = Uuid::new_v4(); + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage candidate"), + 1 + ); + drop(candidate); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("restart read") + .expect("pending import"); + // Reading or discarding a replay batch must not consume its stored anchor. + let mut admitted = recovered.replay_records(LIMITS).expect("replay batch"); + admitted.pop(); + drop(admitted); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("second restart") + .expect("anchor") + .replay_records(LIMITS) + .expect("records") + .len(), + 2 + ); + assert_eq!( + stage_legacy_migration(&disks, &recovered, owner, LIMITS) + .await + .expect("idempotent retry"), + 1 + ); + assert_eq!( + EcstoreDiskAPI::read_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("first source"), + a + ); + assert_eq!( + EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("second source"), + [a, b].concat() + ); + assert!( + super::super::read_committed(&[first, second], LIMITS.max_bytes) + .await + .expect("active reader") + .is_none(), + "pending import must not activate the production snapshot" + ); + } + + #[tokio::test] + async fn migration_source_change_and_capacity_failure_keep_old_commit() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let a = record("a", MrfKind::PartialWrite, None); + let b = record("b", MrfKind::PartialWrite, None); + source(&disk, &a).await; + let original = capture_legacy_migration(&disks, LIMITS).await.expect("initial capture"); + stage_legacy_migration(&disks, &original, owner, LIMITS) + .await + .expect("initial commit"); + let before = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("old commit"); + source(&disk, &b).await; + assert!(matches!( + stage_legacy_migration(&disks, &original, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + let next = capture_legacy_migration(&disks, LIMITS).await.expect("new source capture"); + assert!(matches!( + stage_legacy_migration( + &disks, + &next, + owner, + MigrationLimits { + max_records: 1, + ..LIMITS + } + ) + .await, + Err(MigrationError::Snapshot(SnapshotError::TooLarge)) + )); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("retained commit"), + before + ); + assert_eq!( + stage_legacy_migration(&disks, &next, owner, LIMITS) + .await + .expect("COW successor"), + 2 + ); + let records = recover_pending_migration(&disks, LIMITS) + .await + .expect("successor restart") + .expect("successor") + .replay_records(LIMITS) + .expect("responsibilities"); + assert!( + records.contains(&a) && records.contains(&b), + "successor must inherit old source responsibilities" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("previous slot retained"), + before + ); + } + + #[tokio::test] + async fn migration_torn_inactive_payload_and_manifest_keep_previous_anchor() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, Uuid::new_v4(), LIMITS) + .await + .expect("initial commit"); + let previous = read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes).await.expect("old payload"); + for (path, bytes) in [ + (PAYLOADS[1], b"torn payload".as_slice()), + (COMMITS[1], b"torn manifest".as_slice()), + ] { + install(&disk, path, bytes, LIMITS.max_bytes) + .await + .expect("interrupted inactive write"); + assert!( + recover_pending_migration(&disks, LIMITS).await.is_err(), + "unknown successor responsibility must block recovery" + ); + assert_eq!(read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes).await.expect("old anchor"), previous); + } + } + + #[tokio::test] + async fn migration_missing_corrupt_and_empty_sources_fail_closed() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + assert!(matches!( + capture_legacy_migration(&[Some(disk.clone()), None], LIMITS).await, + Err(MigrationError::CoverageGap) + )); + let empty = capture_legacy_migration(&[Some(disk.clone())], LIMITS) + .await + .expect("observed empty sources"); + assert!(matches!( + stage_legacy_migration(&[Some(disk.clone())], &empty, Uuid::new_v4(), LIMITS).await, + Err(MigrationError::Empty) + )); + source(&disk, b"corrupt").await; + assert!(matches!( + capture_legacy_migration(&[Some(disk)], LIMITS).await, + Err(MigrationError::Invalid) + )); + } + + #[tokio::test] + async fn migration_commit_boundaries_and_lost_response_recover_idempotently() { + for boundary in [ + Boundary::AfterPayload, + Boundary::BeforeManifest, + Boundary::AfterManifest, + Boundary::AfterReadback, + ] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let bytes = record("a", MrfKind::PartialWrite, None); + source(&disk, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("source survives interruption"), + bytes + ); + let recovery = recover_pending_migration(&disks, LIMITS).await; + if matches!(boundary, Boundary::AfterManifest | Boundary::AfterReadback) { + assert!(recovery.expect("committed restart").is_some()); + } else { + assert!( + matches!(recovery, Err(MigrationError::Conflict)), + "uncommitted candidate is not a committed recovery result" + ); + } + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("retry interrupted stage"), + 1 + ); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("restart after retry") + .expect("anchor") + .replay_records(LIMITS) + .expect("records"), + vec![bytes] + ); + } + } + + #[tokio::test] + async fn migration_interrupted_claim_does_not_authorize_takeover() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed anchor"); + install(&disk, CLAIM, b"interrupted writer", 64) + .await + .expect("interrupted claim"); + assert!(matches!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS).await, + Err(MigrationError::Claimed) + )); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("recovery remains read-only") + .expect("anchor") + .replay_records(LIMITS) + .expect("record") + .len(), + 1 + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM) + .await + .expect("claim retained"), + b"interrupted writer".as_slice() + ); + } + + #[tokio::test] + async fn migration_source_change_after_payload_prevents_manifest_publication() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + let changed = record("b", MrfKind::PartialWrite, None); + SOURCE_CHANGES + .lock() + .expect("source fault map") + .insert(owner, (disk.clone(), changed.clone())); + assert!(matches!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("changed source survives"), + changed + ); + assert!( + read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes) + .await + .expect("candidate retained") + .is_some() + ); + } + + #[test] + fn migration_raw_identity_preserves_kind_scope_and_nil_version() { + let id = Uuid::new_v4(); + let mut variants = Vec::new(); + for kind in [MrfKind::PartialWrite, MrfKind::DecodeFailure] { + for scope in [ + None, + Some(MrfScope { + pool_index: 0, + set_index: 0, + }), + Some(MrfScope { + pool_index: 0, + set_index: 1, + }), + ] { + variants.push(record("same", kind, scope)); + } + } + let mut nil = record("same", MrfKind::PartialWrite, None); + nil[12] = 1; + nil.splice(13..13, [0; 16]); + let end = nil.len() - 4; + let mut crc = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + crc.update(&nil[..end]); + nil[end..].copy_from_slice(&u32::try_from(crc.finalize()).expect("CRC").to_le_bytes()); + variants.push(nil); + let bytes = variants.concat(); + let candidate = PendingMigration { + version: 1, + sources: vec![Source { + disk_id: id, + path: LegacyPath::Scoped, + digest: Sha256::digest(&bytes).into(), + bytes: Some(bytes), + }], + inherited: Vec::new(), + }; + let records = candidate.replay_records(LIMITS).expect("raw identities"); + assert_eq!(records.len(), variants.len()); + for variant in variants { + assert!(records.contains(&variant)); + } + } + + async fn slot_bytes(disk: &EcstoreDiskStore) -> Vec>> { + let mut bytes = Vec::new(); + for path in PAYLOADS.into_iter().chain(COMMITS) { + bytes.push(read_bounded(disk, path, LIMITS.max_bytes).await.expect("slot bytes")); + } + bytes + } + + #[tokio::test] + async fn migration_damaged_newer_commit_never_overwrites_successor_responsibility() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed generation"); + } + install(&disk, COMMITS[1], b"torn higher manifest", MANIFEST_LEN) + .await + .expect("manifest fault"); + source(&disk, &record("c", MrfKind::PartialWrite, None)).await; + let before = slot_bytes(&disk).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("latest legacy source"); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert_eq!(slot_bytes(&disk).await, before, "the only payload containing b must not be overwritten"); + } + + #[tokio::test] + async fn migration_lower_limits_never_fall_back_to_a_smaller_old_generation() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed generation"); + } + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let before = slot_bytes(&disk).await; + let first_len = before[0].as_ref().expect("first payload").len(); + for limits in [ + MigrationLimits { + max_bytes: first_len, + ..LIMITS + }, + MigrationLimits { + max_records: 1, + ..LIMITS + }, + MigrationLimits { + max_sources: 2, + ..LIMITS + }, + ] { + assert!(matches!( + recover_pending_migration(&disks, limits).await, + Err(MigrationError::Snapshot(SnapshotError::TooLarge)) + )); + assert_eq!(slot_bytes(&disk).await, before); + } + } + + #[tokio::test] + async fn migration_source_history_count_is_bounded_before_successor_write() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let limits = MigrationLimits { + max_sources: 3, + ..LIMITS + }; + let mut records = (0..10) + .map(|n| record(&n.to_string(), MrfKind::PartialWrite, None)) + .collect::>(); + for round in 0..3 { + records.rotate_left(1); + source(&disk, &records.concat()).await; + let candidate = capture_legacy_migration(&disks, limits) + .await + .expect("bounded current sources"); + assert_eq!(candidate.replay_records(limits).expect("same responsibilities").len(), 10); + let before = slot_bytes(&disk).await; + let result = stage_legacy_migration(&disks, &candidate, owner, limits).await; + if round < 2 { + assert_eq!(result.expect("within source count"), round + 1); + } else { + assert!(matches!(result, Err(MigrationError::Snapshot(SnapshotError::TooLarge)))); + assert_eq!(slot_bytes(&disk).await, before); + } + } + } + + #[tokio::test] + async fn migration_empty_legacy_sources_still_inherit_prior_responsibilities() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let bytes = record("a", MrfKind::PartialWrite, None); + source(&disk, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("initial source"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("initial stage"); + EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + Some(bytes.clone().into()), + None, + ) + .await + .expect("legacy removes source"); + let empty = capture_legacy_migration(&disks, LIMITS) + .await + .expect("valid absent-source observation"); + assert!(empty.replay_records(LIMITS).expect("empty observation").is_empty()); + assert_eq!( + stage_legacy_migration(&disks, &empty, owner, LIMITS) + .await + .expect("inherit earlier obligation"), + 2 + ); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("restart") + .expect("pending") + .replay_records(LIMITS) + .expect("inherited responsibility"), + vec![bytes] + ); + } + + #[tokio::test] + async fn migration_release_failure_still_releases_other_owned_claims() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + source(&first, &record("a", MrfKind::PartialWrite, None)).await; + source(&second, &record("a", MrfKind::PartialWrite, None)).await; + let disks = [Some(first), Some(second)]; + let ordered = configured_disks(&disks).await.expect("disk order"); + let owner = Uuid::new_v4(); + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, Boundary::Release); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert!( + read_bounded(&ordered[0], CLAIM, 64) + .await + .expect("failed release remains claimed") + .is_some() + ); + assert!( + read_bounded(&ordered[1], CLAIM, 64) + .await + .expect("later release attempted") + .is_none() + ); + } + + #[tokio::test] + async fn migration_retry_repairs_missing_manifest_replica() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let disks = [Some(first.clone()), Some(second.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + let bytes = record(name, MrfKind::PartialWrite, None); + source(&first, &bytes).await; + source(&second, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage generation"); + } + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let committed = read_bounded(&second, COMMITS[1], MANIFEST_LEN) + .await + .expect("manifest") + .expect("committed"); + EcstoreDiskAPI::compare_and_update_file( + second.as_ref(), + RUSTFS_META_BUCKET, + COMMITS[1], + Some(committed.clone().into()), + None, + ) + .await + .expect("lost replica"); + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("retry missing replica"), + 2 + ); + assert_eq!( + read_bounded(&second, COMMITS[1], MANIFEST_LEN) + .await + .expect("repaired manifest"), + Some(committed) + ); + } + + #[tokio::test] + async fn migration_old_payload_orphan_matches_any_validated_replica() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let disks = [Some(first.clone()), Some(second)]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + let bytes = record(name, MrfKind::PartialWrite, None); + for disk in disks.iter().flatten() { + source(disk, &bytes).await; + } + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage generation"); + } + let old_manifest = read_bounded(&first, COMMITS[0], MANIFEST_LEN) + .await + .expect("old manifest") + .expect("gen1"); + assert_eq!( + EcstoreDiskAPI::compare_and_update_file( + first.as_ref(), + RUSTFS_META_BUCKET, + COMMITS[0], + Some(old_manifest.into()), + None + ) + .await + .expect("remove one old manifest"), + EcstoreConditionalFileUpdate::Updated + ); + let before = slot_bytes(&first).await; + for ordered in [disks.clone(), [disks[1].clone(), disks[0].clone()]] { + let recovered = recover_pending_migration(&ordered, LIMITS) + .await + .expect("older orphan has independent proof") + .expect("gen2"); + assert_eq!(recovered.replay_records(LIMITS).expect("A and B obligations").len(), 2); + let candidate = capture_legacy_migration(&ordered, LIMITS).await.expect("current B source"); + assert_eq!( + stage_legacy_migration(&ordered, &candidate, owner, LIMITS) + .await + .expect("gen2 retry"), + 2 + ); + assert_eq!( + slot_bytes(&first).await, + before, + "known older orphan must not cause fallback or overwrite" + ); + } + } + + #[tokio::test] + async fn migration_successor_retry_validates_orphan_after_inheriting_previous_records() { + for boundary in [Boundary::AfterPayload, Boundary::BeforeManifest] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let a = record("a", MrfKind::PartialWrite, None); + let b = record("b", MrfKind::PartialWrite, None); + source(&disk, &a).await; + let original = capture_legacy_migration(&disks, LIMITS).await.expect("source A"); + stage_legacy_migration(&disks, &original, owner, LIMITS).await.expect("gen1"); + let old_slot = slot_bytes(&disk).await; + source(&disk, &b).await; + let next = capture_legacy_migration(&disks, LIMITS).await.expect("source B"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &next, owner, LIMITS).await.is_err()); + assert!( + matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict)), + "uncommitted AB is not silently accepted as A" + ); + let captured_again = capture_legacy_migration(&disks, LIMITS) + .await + .expect("restart source capture contains only B"); + assert_eq!(captured_again.replay_records(LIMITS).expect("current source"), vec![b.clone()]); + assert_eq!( + stage_legacy_migration(&disks, &captured_again, owner, LIMITS) + .await + .expect("retry must compare inherited AB"), + 2 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("committed restart") + .expect("gen2"); + let records = recovered.replay_records(LIMITS).expect("retained A and B"); + assert_eq!(records.len(), 2); + assert!(records.contains(&a) && records.contains(&b)); + let after = slot_bytes(&disk).await; + assert_eq!(after[0], old_slot[0]); + assert_eq!(after[2], old_slot[2]); + } + } + + #[tokio::test] + async fn migration_third_generation_retry_keeps_reused_slot_recoverable() { + for boundary in [Boundary::AfterPayload, Boundary::BeforeManifest] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let records = ["a", "b", "c"] + .into_iter() + .map(|name| record(name, MrfKind::PartialWrite, None)) + .collect::>(); + + for bytes in &records[..2] { + source(&disk, bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture committed generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage committed generation"); + } + + source(&disk, &records[2]).await; + let third = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture third generation"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &third, owner, LIMITS).await.is_err()); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + + let retry = capture_legacy_migration(&disks, LIMITS) + .await + .expect("recapture third generation"); + assert_eq!( + stage_legacy_migration(&disks, &retry, owner, LIMITS) + .await + .expect("retry third generation"), + 3 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("recover after third retry") + .expect("third generation"); + let replayed = recovered.replay_records(LIMITS).expect("all staged responsibilities"); + assert_eq!(replayed.len(), 3); + for record in &records { + assert!(replayed.contains(record)); + } + } + } + + #[tokio::test] + async fn migration_third_generation_source_change_retry_keeps_reused_slot_recoverable() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let records = ["a", "b", "c", "d"] + .into_iter() + .map(|name| record(name, MrfKind::PartialWrite, None)) + .collect::>(); + + for bytes in &records[..2] { + source(&disk, bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture committed generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage committed generation"); + } + + source(&disk, &records[2]).await; + let third = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture third generation"); + SOURCE_CHANGES + .lock() + .expect("source fault map") + .insert(owner, (disk.clone(), records[3].clone())); + assert!(matches!( + stage_legacy_migration(&disks, &third, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + + source(&disk, &records[2]).await; + let retry = capture_legacy_migration(&disks, LIMITS) + .await + .expect("recapture restored third generation"); + assert_eq!( + stage_legacy_migration(&disks, &retry, owner, LIMITS) + .await + .expect("retry restored third generation"), + 3 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("recover after restored third retry") + .expect("third generation"); + let replayed = recovered.replay_records(LIMITS).expect("all staged responsibilities"); + assert_eq!(replayed.len(), 3); + for record in &records[..3] { + assert!(replayed.contains(record)); + } + assert!(!replayed.contains(&records[3])); + } + + #[tokio::test] + async fn migration_newer_manifest_with_stale_payload_fails_closed() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("commit"); + } + let old = read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes) + .await + .expect("old read") + .expect("old payload"); + install(&disk, PAYLOADS[1], &old, LIMITS.max_bytes) + .await + .expect("stale payload corruption"); + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let recovery = recover_pending_migration(&disks, LIMITS).await; + assert!( + recovery.is_err(), + "newer manifest must prevent fallback to old responsibilities: {recovery:?}" + ); + } +} diff --git a/crates/heal/src/heal/outcome.rs b/crates/heal/src/heal/outcome.rs new file mode 100644 index 000000000..ada74376d --- /dev/null +++ b/crates/heal/src/heal/outcome.rs @@ -0,0 +1,305 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Execution results are separate from repair responsibility. A legacy +//! successful storage call supplies no authoritative repair receipt. + +use std::{collections::VecDeque, time::SystemTime}; +use uuid::Uuid; + +const MAX_OUTCOME_ITEMS: usize = 128; +const MAX_OUTCOME_BYTES: usize = 64 * 1024; +const MAX_OUTCOME_DETAIL_BYTES: usize = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealObjectKind { + Object, + Metadata, + Decode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HealObjectIdentity { + pub kind: HealObjectKind, + pub bucket: String, + pub object: String, + /// The requested version; None remains unresolved, never an absence proof. + pub version_id: Option, + pub bucket_incarnation_id: Option, + pub pool_index: Option, + pub set_index: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealDeferredReason { + DanglingDeleteGrace, + TransientUsageCache, + TransientExistenceCheck, + Deadline, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealFailureClass { + Recoverable, + RetryExhausted, + Permanent, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HealObjectDisposition { + /// The legacy storage response does not prove the requested check or commit. + Unknown, + Repaired, + VerifiedHealthy, + AuthoritativelyAbsent, + Deferred { + reason: HealDeferredReason, + retry_not_before: Option, + }, + Failed(HealFailureClass), + Cancelled, + DryRunObserved, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HealObjectOutcome { + pub identity: HealObjectIdentity, + pub disposition: HealObjectDisposition, + pub detail: Option, +} + +impl HealObjectOutcome { + fn retained_bytes(&self) -> usize { + size_of::() + .saturating_add(self.identity.bucket.capacity()) + .saturating_add(self.identity.object.capacity()) + .saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity)) + .saturating_add(self.detail.as_ref().map_or(0, String::capacity)) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealTraversalCoverage { + #[default] + Unknown, + Partial, + Complete, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealAbortReason { + Cancelled, + Deadline, + Untraversable, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealExecutionOutcome { + #[default] + Pending, + Running, + Completed, + CompletedWithErrors, + Aborted(HealAbortReason), +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HealOutcomeCounters { + pub processed: u64, + pub healed: u64, + pub unchanged: u64, + /// Deferred, cancelled, dry-run and unverified results remain unresolved. + pub skipped: u64, + pub failed: u64, + pub unknown: u64, + pub attempt_failures: u64, + pub overflowed: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HealTaskOutcome { + pub execution: HealExecutionOutcome, + pub coverage: HealTraversalCoverage, + pub counters: HealOutcomeCounters, + /// A bounded diagnostic window, not a complete responsibility ledger. + pub objects: VecDeque, + pub objects_truncated: bool, + retained_object_bytes: usize, + untraversable: bool, +} + +impl HealTaskOutcome { + pub(crate) fn start(&mut self) { + if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) { + self.execution = HealExecutionOutcome::Running; + } + self.coverage = HealTraversalCoverage::Partial; + } + + pub(crate) fn attempt_failed(&mut self) { + self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures); + } + + pub(crate) fn mark_untraversable(&mut self) { + self.untraversable = true; + self.coverage = HealTraversalCoverage::Partial; + } + + pub(crate) fn finish(&mut self, abort: Option) { + if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) { + return; + } + let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable)); + self.execution = match abort { + Some(reason) => HealExecutionOutcome::Aborted(reason), + None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors, + None => HealExecutionOutcome::Completed, + }; + self.coverage = if abort.is_none() && !self.counters.overflowed { + HealTraversalCoverage::Complete + } else { + HealTraversalCoverage::Partial + }; + } + + pub(crate) fn record(&mut self, mut item: HealObjectOutcome) { + use super::progress::increment_counter; + let counters = &mut self.counters; + counters.overflowed |= !increment_counter(&mut counters.processed); + let counter = match item.disposition { + HealObjectDisposition::Repaired => &mut counters.healed, + HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged, + HealObjectDisposition::Failed(_) => &mut counters.failed, + HealObjectDisposition::Unknown => { + counters.overflowed |= !increment_counter(&mut counters.unknown); + &mut counters.skipped + } + _ => &mut counters.skipped, + }; + counters.overflowed |= !increment_counter(counter); + if let Some(detail) = &mut item.detail { + let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES); + while !detail.is_char_boundary(end) { + end -= 1; + } + self.objects_truncated |= end < detail.len(); + detail.truncate(end); + detail.shrink_to_fit(); + } + let bytes = item.retained_bytes(); + if bytes > MAX_OUTCOME_BYTES { + self.objects_truncated = true; + return; + } + while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES { + let Some(oldest) = self.objects.pop_front() else { break }; + self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes()); + self.objects_truncated = true; + } + self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes); + self.objects.push_back(item); + } + + pub(crate) fn retained_bytes(&self) -> usize { + size_of::() + .saturating_add(self.retained_object_bytes) + .saturating_add(self.objects.capacity().saturating_mul(size_of::())) + } +} + +#[cfg(test)] +mod canonical_outcome_tests { + use super::*; + + fn item(disposition: HealObjectDisposition) -> HealObjectOutcome { + HealObjectOutcome { + identity: HealObjectIdentity { + kind: HealObjectKind::Object, + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + bucket_incarnation_id: None, + pool_index: None, + set_index: None, + }, + disposition, + detail: None, + } + } + + #[test] + fn canonical_outcome_categories_have_one_terminal_count() { + let mut outcome = HealTaskOutcome::default(); + for disposition in [ + HealObjectDisposition::Unknown, + HealObjectDisposition::Repaired, + HealObjectDisposition::VerifiedHealthy, + HealObjectDisposition::AuthoritativelyAbsent, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + retry_not_before: None, + }, + HealObjectDisposition::Failed(HealFailureClass::Permanent), + HealObjectDisposition::Cancelled, + HealObjectDisposition::DryRunObserved, + ] { + outcome.record(item(disposition)); + } + let c = &outcome.counters; + assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1)); + assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed); + } + + #[test] + fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() { + let mut outcome = HealTaskOutcome::default(); + for _ in 0..MAX_OUTCOME_ITEMS { + outcome.record(item(HealObjectDisposition::Unknown)); + } + assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS); + assert!(!outcome.objects_truncated); + outcome.record(item(HealObjectDisposition::Unknown)); + assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS); + assert!(outcome.objects_truncated); + let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent)); + oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES); + outcome.record(oversized); + assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count")); + assert_eq!(outcome.counters.failed, 1); + assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES); + for _ in 0..MAX_OUTCOME_ITEMS { + let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent)); + failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES)); + outcome.record(failed); + } + assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES); + assert!(outcome.objects.iter().all(|item| { + item.detail + .as_ref() + .is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES) + })); + assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS); + } + + #[test] + fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() { + let mut outcome = HealTaskOutcome::default(); + outcome.counters.processed = u64::MAX; + outcome.record(item(HealObjectDisposition::Unknown)); + outcome.finish(None); + assert!(outcome.counters.overflowed); + assert_eq!(outcome.counters.processed, u64::MAX); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + } +} diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index c4c1d902b..a1b09c3b7 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -15,6 +15,10 @@ use crate::heal::{ DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _, erasure_healer::target_outcomes_complete, + outcome::{ + HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind, + HealObjectOutcome, HealTaskOutcome, + }, progress::HealProgress, resume::{ CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, @@ -43,6 +47,26 @@ use uuid::Uuid; use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET}; +#[cfg(test)] +pub(crate) struct OutcomeFinishTestHook { + pub(crate) task_id: String, + pub(crate) reached: tokio::sync::Notify, + pub(crate) release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock>>> = + std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None)); + +#[cfg(test)] +async fn pause_outcome_finish(task_id: &str) { + let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone(); + if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) { + hook.reached.notify_one(); + hook.release.notified().await; + } +} + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_TASK: &str = "task"; const LOG_SUBSYSTEM_OBJECT: &str = "object"; @@ -394,6 +418,7 @@ pub struct HealTask { pub status: Arc>, /// Progress tracking pub progress: Arc>, + outcome: Arc>, /// Result items collected from storage heal calls, each stamped with a /// monotonically increasing sequence number for incremental consumption /// (the client passes the last seen seq back and receives only newer @@ -460,6 +485,7 @@ impl HealTask { result_items_truncated: Arc::new(AtomicBool::new(false)), batch_failure: Arc::new(RwLock::new(None)), batch_failure_recorded: Arc::new(AtomicBool::new(false)), + outcome: Arc::new(RwLock::new(HealTaskOutcome::default())), created_at: request.created_at, enqueued_at: request.enqueued_at, started_at: Arc::new(RwLock::new(None)), @@ -507,6 +533,66 @@ impl HealTask { self.heal_type.kind_label() } + pub async fn get_outcome(&self) -> HealTaskOutcome { + self.outcome.read().await.clone() + } + + fn outcome_identity( + &self, + bucket: &str, + object: &str, + version_id: Option<&str>, + pool_index: Option, + set_index: Option, + ) -> HealObjectIdentity { + HealObjectIdentity { + kind: match self.heal_type { + HealType::Metadata { .. } => HealObjectKind::Metadata, + HealType::ECDecode { .. } => HealObjectKind::Decode, + _ => HealObjectKind::Object, + }, + bucket: bucket.to_owned(), + object: object.to_owned(), + version_id: version_id.map(ToOwned::to_owned), + bucket_incarnation_id: None, + pool_index, + set_index, + } + } + + fn single_object_identity(&self) -> Option { + let (bucket, object, version) = match &self.heal_type { + HealType::Object { + bucket, + object, + version_id, + } + | HealType::ECDecode { + bucket, + object, + version_id, + } => (bucket, object, version_id.as_deref()), + HealType::Metadata { bucket, object } => (bucket, object, None), + _ => return None, + }; + Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index)) + } + + async fn record_deferred_object(&self, reason: HealDeferredReason) { + if let Some(identity) = self.single_object_identity() { + let mut outcome = self.outcome.write().await; + outcome.attempt_failed(); + outcome.record(HealObjectOutcome { + identity, + disposition: HealObjectDisposition::Deferred { + reason, + retry_not_before: None, + }, + detail: None, + }); + } + } + pub(crate) fn has_batch_failure(&self) -> bool { self.batch_failure_recorded.load(Ordering::Acquire) } @@ -634,6 +720,7 @@ impl HealTask { } async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> { + self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await; warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -733,6 +820,8 @@ impl HealTask { return false; } + self.record_deferred_object(HealDeferredReason::TransientUsageCache).await; + warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -755,6 +844,8 @@ impl HealTask { return false; } + self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await; + warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -801,6 +892,7 @@ impl HealTask { #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))] #[hotpath::measure] pub async fn execute(&self) -> Result<()> { + self.outcome.write().await.start(); // update status and timestamps atomically to avoid race conditions let now = SystemTime::now(); let start_instant = Instant::now(); @@ -860,6 +952,45 @@ impl HealTask { HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await, }; + #[cfg(test)] + pause_outcome_finish(&self.id).await; + { + let mut outcome = self.outcome.write().await; + if outcome.counters.processed == 0 + && let Some(identity) = self.single_object_identity() + { + let disposition = match &result { + Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved, + Ok(()) => HealObjectDisposition::Unknown, + Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled, + Err(Error::TaskTimeout) => HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + }, + Err(error) => { + outcome.attempt_failed(); + HealObjectDisposition::Failed(if error.is_recoverable_heal() { + HealFailureClass::Recoverable + } else { + HealFailureClass::Permanent + }) + } + }; + outcome.record(HealObjectOutcome { + identity, + disposition, + detail: result.as_ref().err().map(ToString::to_string), + }); + } + let abort = match &result { + Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled), + Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline), + Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable), + _ => None, + }; + outcome.finish(abort); + } + // update completed time and status { let mut completed_at = self.completed_at.write().await; @@ -944,6 +1075,7 @@ impl HealTask { pub async fn cancel(&self) -> Result<()> { self.cancel_token.cancel(); + self.outcome.write().await.finish(Some(HealAbortReason::Cancelled)); let mut status = self.status.write().await; *status = HealTaskStatus::Cancelled; debug!( diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index d44df549e..c6225c844 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -214,6 +214,7 @@ impl HealTask { continue; } failed = failed.saturating_add(1); + self.outcome.write().await.mark_untraversable(); if err.is_recoverable_heal() { retryable = retryable.saturating_add(1); } else { @@ -260,6 +261,7 @@ impl HealTask { #[hotpath::measure] async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> { + let previous_progress = self.get_progress().await; let mut scanned = 0u64; let mut healed = 0u64; let mut failed = 0u64; @@ -304,23 +306,47 @@ impl HealTask { let mut continuation_token: Option = None; loop { self.check_control_flags().await?; - let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() { - self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( - set_disk_id, - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await? - } else { - self.await_with_control(self.storage.list_objects_for_heal_page( - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await? + let mut listing_attempt = 0; + let (objects, next_token, is_truncated) = loop { + let page = if let Some(set_disk_id) = set_disk_id.as_deref() { + self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await + } else { + self.await_with_control(self.storage.list_objects_for_heal_page( + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await + }; + match page { + Ok(page) => break page, + Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error), + Err(error) => { + self.outcome.write().await.attempt_failed(); + if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES { + listing_attempt += 1; + self.await_with_control(async { + tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await; + Ok(()) + }) + .await?; + continue; + } + self.outcome.write().await.mark_untraversable(); + return Err(Error::HealListingFailed { + bucket: bucket.to_string(), + source: Box::new(error), + }); + } + } }; let mut pending = objects; @@ -338,6 +364,14 @@ impl HealTask { self.check_control_flags().await?; let mut telemetry_unknown = false; let object = item.name.as_str(); + let identity = + self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set); + let mut disposition = if heal_opts.dry_run { + HealObjectDisposition::DryRunObserved + } else { + HealObjectDisposition::Unknown + }; + let mut detail = None; { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("{bucket}/{object}"))); @@ -380,7 +414,31 @@ impl HealTask { }; if let Some(err) = error { + match err { + Error::TaskCancelled | Error::TaskTimeout => { + let disposition = if matches!(err, Error::TaskCancelled) { + HealObjectDisposition::Cancelled + } else { + HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + } + }; + self.outcome.write().await.record(HealObjectOutcome { + identity, + disposition, + detail: None, + }); + return Err(err); + } + _ => self.outcome.write().await.attempt_failed(), + } + detail = Some(err.to_string()); if Self::is_dangling_delete_grace_error(&err) { + disposition = HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + retry_not_before: None, + }; telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", @@ -395,6 +453,10 @@ impl HealTask { "Heal bucket object dangling cleanup deferred by grace window" ); } else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) { + disposition = HealObjectDisposition::Deferred { + reason: HealDeferredReason::TransientUsageCache, + retry_not_before: None, + }; telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", @@ -425,6 +487,11 @@ impl HealTask { ); retry.push(item); } else { + disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() { + HealFailureClass::RetryExhausted + } else { + HealFailureClass::Permanent + }); telemetry_unknown |= !increment_counter(&mut failed); if err.is_recoverable_heal() { retryable_failed = retryable_failed.saturating_add(1); @@ -459,8 +526,20 @@ impl HealTask { continue; } + self.outcome.write().await.record(HealObjectOutcome { + identity, + disposition, + detail, + }); + let mut progress = self.progress.write().await; - progress.update_object_progress(scanned, healed, failed, skipped, bytes); + progress.update_object_progress( + previous_progress.objects_scanned.saturating_add(scanned), + previous_progress.objects_healed.saturating_add(healed), + previous_progress.objects_failed.saturating_add(failed), + previous_progress.skipped_objects.saturating_add(skipped), + previous_progress.bytes_processed.saturating_add(bytes), + ); if telemetry_unknown { progress.mark_unknown(); } @@ -475,7 +554,7 @@ impl HealTask { continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; if continuation_token.is_none() { - // Truncated but no continuation token: end of listing. + // Truncated without a continuation token is a compatibility EOF. break; } } diff --git a/crates/heal/src/heal/task/heal_metadata.rs b/crates/heal/src/heal/task/heal_metadata.rs index fe2dc98ac..fad5a403e 100644 --- a/crates/heal/src/heal/task/heal_metadata.rs +++ b/crates/heal/src/heal/task/heal_metadata.rs @@ -261,8 +261,8 @@ impl HealTask { update_parity: true, no_lock: self.options.no_lock, read_repair: false, - pool: None, - set: None, + pool: self.options.pool_index, + set: self.options.set_index, }; let heal_result = self diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 1734d0c7a..8cabf1214 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -14,6 +14,364 @@ use super::super::{DiskOption, DiskStore, Endpoint, new_disk}; use super::*; + +mod canonical_outcome { + use super::*; + use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + + fn bucket_task(storage: Arc) -> HealTask { + HealTask::from_request( + HealRequest::new( + HealType::Bucket { + bucket: "bucket-a".to_string(), + }, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage, + ) + } + + #[tokio::test(start_paused = true)] + async fn cluster_retries_only_the_failed_listing_page() { + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(1)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect("second-page retry succeeds"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.attempt_failures, 1); + assert_eq!(task.get_progress().await.objects_scanned, 2); + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a", "object-b"] + ); + assert_eq!( + storage.listing_tokens.lock().expect("listing tokens").as_slice(), + [None, Some("second".to_string()), Some("second".to_string())] + ); + } + + #[tokio::test(start_paused = true)] + async fn exhausted_listing_page_cannot_restart_the_bucket() { + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(4)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect_err("listing page budget exhausted"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 1); + assert_eq!(outcome.counters.attempt_failures, 4); + assert_eq!(task.get_progress().await.objects_scanned, 1); + assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]); + assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]); + } + + #[tokio::test] + async fn listing_failure_preserves_processed_objects_and_partial_coverage() { + let storage = Arc::new(MockStorage { + fail_second_listing_page: true, + ..Default::default() + }); + let task = bucket_task(storage); + task.execute().await.expect_err("second page cannot be traversed"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 1); + assert_eq!(outcome.objects[0].identity.object, "object-a"); + assert_eq!(task.get_progress().await.objects_scanned, 1); + } + + #[tokio::test] + async fn cluster_preserves_cumulative_progress_across_buckets() { + let storage = Arc::new(MockStorage { + list_each_bucket: true, + listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage, + ); + task.execute().await.expect("both buckets complete"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.processed, 4); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + let progress = task.get_progress().await; + assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4)); + assert_eq!( + outcome + .objects + .iter() + .filter(|item| item.identity.bucket == "bucket-b") + .count(), + 2 + ); + } + + #[tokio::test(start_paused = true)] + async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() { + let storage = Arc::new(MockStorage::default()); + storage.heal_object_outcomes.lock().expect("outcomes").insert( + "object-a".to_string(), + (0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(), + ); + let task = bucket_task(storage.clone()); + task.execute().await.expect_err("legacy adapter retains batch failure"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1)); + assert_eq!(outcome.counters.attempt_failures, 4); + let failed = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("failed object"); + assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)); + let object_b_calls = { + let calls = storage.heal_object_calls.lock().expect("calls"); + calls.iter().filter(|object| object.as_str() == "object-b").count() + }; + assert_eq!(object_b_calls, 1); + let progress = task.get_progress().await; + assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); + } + + #[tokio::test(start_paused = true)] + async fn retry_success_counts_one_terminal_outcome() { + let storage = Arc::new(MockStorage::default()); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum])); + let task = bucket_task(storage); + task.execute().await.expect("retry should recover"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.failed, 0); + assert_eq!(outcome.counters.attempt_failures, 1); + assert_eq!( + outcome + .objects + .iter() + .filter(|item| item.identity.object == "object-a") + .count(), + 1 + ); + assert_eq!( + outcome.counters.processed, + outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed + ); + } + + #[tokio::test] + async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() { + let storage = Arc::new(MockStorage::default()); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred])); + let task = bucket_task(storage); + task.execute().await.expect("grace permits traversal completion"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt"); + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + .. + } + )); + assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown); + assert!( + outcome + .objects + .iter() + .all(|item| item.identity.bucket_incarnation_id.is_none()) + ); + assert_eq!( + task.get_progress().await.objects_healed, + 1, + "legacy display count remains distinct from proof" + ); + } + + #[tokio::test] + async fn grace_single_object_is_completed_but_deferred() { + let storage = Arc::new(MockStorage { + heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)), + ..Default::default() + }); + let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage); + task.execute().await.expect("grace is deferred"); + let outcome = task.get_outcome().await; + assert_eq!(task.get_status().await, HealTaskStatus::Completed); + assert_eq!(outcome.counters.processed, 1); + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + .. + } + )); + assert_eq!(outcome.counters.attempt_failures, 1); + } + + #[tokio::test] + async fn dry_run_and_transient_existence_do_not_prove_repair() { + for transient in [false, true] { + let storage = Arc::new(MockStorage::default()); + if transient { + storage + .object_exists_by_name + .lock() + .expect("existence fixture") + .insert("object".to_string(), MockObjectExists::TransientSkip("retry later")); + } + let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None); + request.options.dry_run = !transient; + let task = HealTask::from_request(request, storage); + task.execute().await.expect("observation may complete"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.healed, 0); + if transient { + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::TransientExistenceCheck, + .. + } + )); + } else { + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved); + } + } + } + + #[tokio::test] + async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])), + bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect_err("structural bucket error"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!( + storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), + ["bucket-a", "bucket-b"] + ); + } + + #[tokio::test(start_paused = true)] + async fn cancellation_and_deadline_leave_partial_coverage() { + for cancel in [false, true] { + let storage = Arc::new(MockStorage { + block_heal_object: Mutex::new(true), + ..Default::default() + }); + let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None); + request.options.timeout = Some(Duration::from_secs(1)); + let task = HealTask::from_request(request, storage); + if cancel { + task.cancel().await.expect("cancel request"); + } + task.execute().await.expect_err("control interruption"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!( + outcome.execution, + HealExecutionOutcome::Aborted(if cancel { + HealAbortReason::Cancelled + } else { + HealAbortReason::Deadline + }) + ); + } + } + + #[tokio::test] + async fn decode_keeps_the_requested_pool_and_set() { + let storage = Arc::new(MockStorage::default()); + let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string())); + request.options.pool_index = Some(2); + request.options.set_index = Some(3); + let task = HealTask::from_request(request, storage.clone()); + task.execute().await.expect("decode fixture"); + let pool_and_set = { + let options = storage.object_heal_opts.lock().expect("storage options"); + (options[0].pool, options[0].set) + }; + assert_eq!(pool_and_set, (Some(2), Some(3))); + let outcome = task.get_outcome().await; + let identity = &outcome.objects[0].identity; + assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3))); + assert_eq!(identity.version_id.as_deref(), Some("version-a")); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown); + } +} use crate::heal::storage::{HealListItem, HealObjectInfo}; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; @@ -582,6 +940,10 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() { #[derive(Default)] struct MockStorage { listed: Mutex, + list_each_bucket: bool, + fail_second_listing_page: bool, + recoverable_second_page_failures: Mutex>, + listing_tokens: Mutex>>, healed_objects: Mutex>, heal_object_calls: Mutex>, heal_object_version_ids: Mutex>>, @@ -995,12 +1357,41 @@ impl HealStorageAPI for MockStorage { _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.listed_prefixes.lock().unwrap().push(prefix.to_string()); + self.listing_tokens + .lock() + .expect("listing tokens") + .push(continuation_token.map(ToOwned::to_owned)); + if let Some(remaining) = self + .recoverable_second_page_failures + .lock() + .expect("listing failures") + .as_mut() + { + if continuation_token.is_none() { + return Ok((vec![heal_item("object-a")], Some("second".to_string()), true)); + } + if *remaining > 0 { + *remaining -= 1; + return Err(Error::Storage(EcstoreError::InsufficientReadQuorum( + bucket.to_string(), + "page".to_string(), + ))); + } + return Ok((vec![heal_item("object-b")], None, false)); + } + if self.fail_second_listing_page { + return if continuation_token.is_none() { + Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true)) + } else { + Err(Error::other("listing unavailable")) + }; + } if *self.truncate_without_token.lock().unwrap() { return Ok((vec![heal_item("object-a")], None, true)); } let mut listed = self.listed.lock().unwrap(); - if continuation_token.is_none() && !*listed { + if continuation_token.is_none() && (!*listed || self.list_each_bucket) { *listed = true; let objects = if bucket == RUSTFS_META_BUCKET { vec![ @@ -1393,6 +1784,8 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() { #[tokio::test] async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() { + use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + // A version listing can report the final page as truncated with no // continuation token. That is treated as end-of-listing (not an error), // so the returned page is healed and the pass terminates cleanly instead @@ -1414,10 +1807,16 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() { ); let task = HealTask::from_request(request, storage.clone()); - task.heal_bucket("bucket-a") + task.execute() .await .expect("truncated-without-token must terminate cleanly, not loop or error"); + assert_eq!(task.get_status().await, HealTaskStatus::Completed); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 1); + assert_eq!( storage.healed_objects.lock().unwrap().as_slice(), ["object-a".to_string()], diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index e57a6cc1a..b9de1c073 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1888,9 +1888,9 @@ where // A remote restart or movement flip invalidates // the token proof; usage_store interprets this // as a publication barrier and performs no PUT. - return true; + return Some(ScannerCycleDeferReason::DataMovement); } - storeapi.scanner_data_usage_publication_blocked().await + scanner_local_publication_defer_reason(storeapi.as_ref()).await } }, ) @@ -3239,8 +3239,8 @@ where { match status { ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => { - if storeapi.scanner_data_usage_publication_blocked().await { - return Some(ScannerCycleDeferReason::DataMovement); + if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await { + return Some(reason); } if status == ScannerCycleStatus::Complete { let distributed = storeapi.setup_is_dist_erasure().await; @@ -3263,6 +3263,22 @@ where } } +async fn scanner_local_publication_defer_reason(storeapi: &S) -> Option +where + S: ScannerStorage, +{ + if !storeapi.scanner_data_usage_publication_blocked().await { + return None; + } + // Pending namespace commits invalidate this publication attempt, but only + // storage movement creates durable, rate-limited catch-up debt. + if storeapi.scanner_data_movement_pause_status().await.paused { + Some(ScannerCycleDeferReason::DataMovement) + } else { + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) + } +} + fn scanner_post_lease_activity_defer_reason( expected_digest: Option<[u8; 32]>, activity: Result, diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index dbbbdca62..37d45aa89 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -266,6 +266,11 @@ async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() { } let pause_status = store.scanner_data_movement_pause_status().await; assert!(pause_status.paused); + assert_eq!( + scanner_local_publication_defer_reason(store.as_ref()).await, + Some(ScannerCycleDeferReason::DataMovement), + "an actual data-movement pause must retain durable catch-up tracking" + ); paused_probe.wait().await; drop(paused_probe); @@ -1171,6 +1176,9 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let (_temp_dir, store) = setup_scanner_cycle_store().await; + let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now()) + .await + .expect("scanner pause backlog should be available"); let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple()); store .make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default()) @@ -1195,6 +1203,13 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin .await .expect("fixture usage baseline should be readable"); let pending = ecstore_hold_namespace_commit(store.as_ref()); + assert_eq!( + scanner_local_publication_defer_reason(store.as_ref()).await, + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), + "an ordinary namespace commit must not be classified as data movement" + ); + let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + assert_eq!(pause_backlog_attempt, ScannerPauseBacklogAttemptDecision::Untracked); let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); let mut cycle_info = CurrentCycle { @@ -1209,7 +1224,15 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin .await .expect("the coordinator must finish its namespace walk while a PUT is pending"); assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal"); - assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert_eq!( + outcome, + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &store, pause_backlog_attempt, outcome).await; + let pause_backlog_status = scanner_pause_backlog_status(store.clone()).await; + assert_eq!(pause_backlog_status.phase, ScannerPauseBacklogPhase::Idle); + assert!(!pause_backlog_status.pending_full_scan); + assert_eq!(pause_backlog_status.catch_up_attempts, 0); assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle"); assert_eq!(revision, DataUsageCacheRevision::Missing); assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before); @@ -5826,7 +5849,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier let probe_calls = route_probe_calls.clone(); async move { let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - route_blocked && call > 1 + (route_blocked && call > 1).then_some(ScannerCycleDeferReason::DataMovement) } }, ) @@ -5872,16 +5895,19 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { data: None, revision: DataUsageCacheRevision::Missing, }), - || async { true }, + || async { Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) }, ) .await; - assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert_eq!( + outcome, + DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); assert!(!store.objects.lock().await.contains_key(&target_key)); assert_eq!( store.put_counts.lock().await.get(&target_key), None, - "the final pool-state fence must run before the first PUT" + "the final publication fence must run before the first PUT" ); } } @@ -5902,7 +5928,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing( receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -5950,7 +5976,7 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -5991,7 +6017,7 @@ async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() { receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -6051,7 +6077,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() { data: Some(Bytes::from(snapshot_data)), revision: DataUsageCacheRevision::Etag("memory-1".to_string()), }), - || async { true }, + || async { Some(ScannerCycleDeferReason::DataMovement) }, ) .await; @@ -6091,7 +6117,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() { // Model the remote lease holder flipping its movement generation // after the activity probe but before the coordinator's PUT. route_store.publication_admission_blocked.store(true, Ordering::Release); - false + None } }, ) @@ -6129,7 +6155,7 @@ async fn coordinator_classifies_an_expired_publication_lease() { revision: DataUsageCacheRevision::Missing, }), ScannerPublicationFence::new(None, Some(expired), None), - || async { false }, + || async { None }, ) .await; @@ -6208,7 +6234,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() { data: None, revision: DataUsageCacheRevision::Missing, }), - || async { true }, + || async { Some(ScannerCycleDeferReason::DataMovement) }, ) .await; diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index af7c616ee..08a3c9996 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -265,7 +265,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel receiver, leader_epoch, initial_baseline, - || async { false }, + || async { None }, ) .await } @@ -280,7 +280,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( ctx, @@ -308,7 +308,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence( ctx, @@ -336,7 +336,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { let ScannerPublicationFence { expected_publication_epoch, @@ -374,18 +374,19 @@ where } else { DATA_USAGE_OBJ_NAME_PATH.as_str() }; - if route_probe().await { + if let Some(reason) = route_probe().await { debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_blocked_before_reconcile", - "Scanner data usage publication deferred by the pool-state fence" + reason = reason.as_str(), + path = %target_path, + "Scanner data usage publication deferred by the publication fence" ); - global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str()); - outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + global_metrics().record_scanner_usage_deferred(reason.as_str()); + outcome = DataUsagePersistOutcome::Deferred(reason); break; } @@ -626,17 +627,18 @@ where if ctx.is_cancelled() { break 'updates; } - if route_probe().await { + if let Some(reason) = route_probe().await { debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_blocked_before_save", - "Scanner data usage publication deferred by the final pool-state fence" + reason = reason.as_str(), + path = %target_path, + "Scanner data usage publication deferred by the final publication fence" ); - break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break DataUsagePersistOutcome::Deferred(reason); } if remote_lease_expired(remote_lease_deadline) { break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded); @@ -722,19 +724,19 @@ where ); } Err(e @ EcstoreError::ObjectNotFound(_, _)) => { - let route_blocked = route_probe().await; - if route_blocked { + if let Some(reason) = route_probe().await { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_deferred", + reason = reason.as_str(), + path = %target_path, error = %e, - "Scanner data usage route is blocked by data movement; retrying later" + "Scanner data usage route remains blocked; retrying later" ); - break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break DataUsagePersistOutcome::Deferred(reason); } error!( target: "rustfs::scanner", diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 410cf6b48..0c2a8c455 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1294,6 +1294,8 @@ impl FolderScanner { } Err(e) => return Err(ScannerError::Io(e)), }; + #[cfg(test)] + tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget); pending_entry_progress = pending_entry_progress.saturating_add(1); if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH || last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 8e26429cc..d47b17acc 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; mod checkpoint_fixture; +pub(super) mod enumeration_restart; /// Reset the process-global alert cooldown map; test-only. fn reset_alert_cooldowns() { diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs index eb10f02e6..7052e9de4 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -20,6 +20,8 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest}; use std::io::Cursor; use tokio::io::AsyncReadExt; +mod segment_observation; + const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; const STATIC_OBJECTS: u64 = 24; const MAX_CACHE_BYTES: u64 = 1024 * 1024; diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs new file mode 100644 index 000000000..b54f36665 --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs @@ -0,0 +1,202 @@ +//! Fixture-only range diagnostics. No result is supplied to a scan selector. + +use super::*; +use std::collections::BTreeSet; + +const MAX_SEGMENTS: usize = 4; +const MAX_SEGMENT_BYTES: usize = 128; +const MAX_WALK_SAMPLES: usize = 32; +const MAX_WALK_BYTES: usize = 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProposalError { + EntryLimit, + ByteLimit, + InvalidKey, +} + +// Keys come from successful fixture writes, not a production mutation stream. +fn fixture_proposal(keys: &[&str]) -> Result, ProposalError> { + let mut segments = BTreeSet::new(); + let mut bytes = 0; + for key in keys { + if key.is_empty() || key.contains(['\\', '\0']) || key.split('/').any(|part| matches!(part, "" | "." | "..")) { + return Err(ProposalError::InvalidKey); + } + let segment = key.split('/').next().expect("validated nonempty key"); + if segments.contains(segment) { + continue; + } + if segments.len() == MAX_SEGMENTS { + return Err(ProposalError::EntryLimit); + } + if segment.len() > MAX_SEGMENT_BYTES - bytes { + return Err(ProposalError::ByteLimit); + } + bytes += segment.len(); + segments.insert(segment.to_string()); + } + Ok(segments) +} + +#[test] +fn segment_observation_fixture_proposal_bounds() { + assert_eq!(fixture_proposal(&["hot/one", "hot/two"]), Ok(BTreeSet::from(["hot".to_string()]))); + assert_eq!(fixture_proposal(&["a", "b", "c", "d"]).expect("entry boundary").len(), MAX_SEGMENTS); + assert_eq!(fixture_proposal(&["a", "b", "c", "d", "e"]), Err(ProposalError::EntryLimit)); + let exact = "x".repeat(MAX_SEGMENT_BYTES); + assert!(fixture_proposal(&[&exact]).is_ok()); + assert_eq!(fixture_proposal(&[&exact, "y"]), Err(ProposalError::ByteLimit)); + let oversized = "x".repeat(MAX_SEGMENT_BYTES + 1); + assert_eq!(fixture_proposal(&[&oversized]), Err(ProposalError::ByteLimit)); + for key in ["", "/hot", "hot/../cold", "hot//one", "hot\\one", "hot/\0"] { + assert_eq!(fixture_proposal(&[key]), Err(ProposalError::InvalidKey)); + } +} + +fn cache_value(cache: &DataUsageCache) -> serde_json::Value { + let mut value = serde_json::to_value(cache).expect("serialize the entire cache"); + // Children are a HashSet: canonicalize only that unordered field, without + // discarding any cache fields or changing ordered histogram arrays. + for (path, entry) in &cache.cache { + value["cache"][path]["children"] = + serde_json::to_value(entry.children.iter().collect::>()).expect("canonical child set"); + } + value +} + +async fn walk_and_save(observe: bool) -> (Vec, serde_json::Value) { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for prefix in ["hot", "cold", "other"] { + for leaf in ["one", "two"] { + let object = format!("{prefix}/{leaf}"); + let mut metadata = FileMeta::new(); + let mut info = FileInfo::new(&object, 4, 2); + info.volume = "bucket".to_string(); + info.name = object.clone(); + info.size = 1; + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp")); + info.metadata.insert("etag".to_string(), "before".to_string()); + metadata.add_version(info).expect("construct segment fixture metadata"); + write_test_object_metadata_bytes(&root, "bucket", &object, &metadata.marshal_msg().expect("encode metadata")).await; + } + } + let changed_key = "hot/one"; + let changed_path = root.join("bucket").join(changed_key).join("xl.meta"); + let before = tokio::fs::read(&changed_path).await.expect("read initial hot metadata"); + let mut metadata = FileMeta::new(); + let mut info = FileInfo::new(changed_key, 4, 2); + info.volume = "bucket".to_string(); + info.name = changed_key.to_string(); + info.size = 1; + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp")); + info.metadata.insert("etag".to_string(), "after!".to_string()); + metadata.add_version(info).expect("construct same-size hot mutation"); + write_test_object_metadata_bytes(&root, "bucket", changed_key, &metadata.marshal_msg().expect("encode hot mutation")).await; + let after = tokio::fs::read(&changed_path) + .await + .expect("read back committed fixture mutation"); + assert_eq!(before.len(), after.len(), "fixture rewrite must keep metadata byte length unchanged"); + assert_ne!(before, after, "a changed key requires an observable successful fixture write"); + scanner.old_cache.info.name = "bucket".to_string(); + scanner.new_cache.info.name = "bucket".to_string(); + scanner.update_cache.info.name = "bucket".to_string(); + let paths = Arc::new(Mutex::new(Vec::::new())); + let proposed_walked = Arc::new(Mutex::new(BTreeSet::::new())); + scanner.update_current_path = Arc::new({ + let paths = paths.clone(); + let proposed_walked = proposed_walked.clone(); + move |path: &str| { + let mut paths = paths.lock().expect("lock bounded actual-walk samples"); + assert!(paths.len() < MAX_WALK_SAMPLES, "fixture walk exceeded its entry budget"); + let bytes: usize = paths.iter().map(String::len).sum(); + assert!(path.len() <= MAX_WALK_BYTES - bytes, "fixture walk exceeded its byte budget"); + paths.push(path.to_string()); + if observe { + let proposed = fixture_proposal(&[changed_key]).expect("bounded successful fixture mutation"); + if let Some(segment) = path.strip_prefix("bucket/").and_then(|path| path.split('/').next()) + && proposed.contains(segment) + { + proposed_walked + .lock() + .expect("lock bounded observed segments") + .insert(segment.to_string()); + } + } + Box::pin(async {}) + } + }); + scanner + .scan_folder( + CancellationToken::new(), + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut DataUsageEntry::default(), + ) + .await + .expect("actual folder walker must finish independently of diagnostics"); + let paths = paths.lock().expect("read walk samples").clone(); + assert!(!paths.is_empty()); + for prefix in ["hot", "cold", "other"] { + assert!( + paths.iter().any(|path| path == &format!("bucket/{prefix}")), + "all fixture segments must actually be walked" + ); + } + let store = FixtureStore::new(); + let revisions = DataUsageCache::default() + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("read empty fixture revisions"); + scanner + .new_cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("save actual walker output through the cache codec and revision gate"); + let loaded = store.strict_load().await; + assert_eq!(loaded.checked_flatten("bucket").expect("complete fixture tree").objects, 6); + assert_eq!( + cache_value(&loaded), + cache_value(&scanner.new_cache), + "codec round-trip must retain the entire cache, not just aggregate size" + ); + if observe { + let proposed = proposed_walked.lock().expect("read callback observations").clone(); + assert_eq!(proposed, BTreeSet::from(["hot".to_string()])); + let walked_segments: BTreeSet<_> = paths + .iter() + .filter_map(|path| path.strip_prefix("bucket/")) + .filter_map(|path| path.split('/').next()) + .collect(); + assert_eq!(walked_segments, BTreeSet::from(["cold", "hot", "other"])); + assert!(proposed.iter().all(|segment| walked_segments.contains(segment.as_str()))); + assert_eq!( + walked_segments.len() - proposed.len(), + 2, + "the two non-proposed segments must still be walked" + ); + eprintln!( + "segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified", + paths.len() + ); + } else { + assert!(proposed_walked.lock().expect("read disabled observations").is_empty()); + } + // Compare semantic values because map encoding order is not content identity. + (paths, cache_value(&loaded)) +} + +#[tokio::test] +#[serial] +async fn segment_observation_on_off_preserves_actual_walk_and_saved_cache() { + let off = walk_and_save(false).await; + let on = walk_and_save(true).await; + assert_eq!(off.0, on.0, "diagnostics must not change actual traversal order or coverage"); + assert_eq!(off.1, on.1, "diagnostics must not change the saved cache result"); +} diff --git a/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs b/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs new file mode 100644 index 000000000..90a1b30eb --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs @@ -0,0 +1,181 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use super::*; +use std::path::{Path, PathBuf}; +use tokio::io::AsyncReadExt; + +const MAX_CACHE_BYTES: u64 = 1024 * 1024; +const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST"; + +struct Observation { + root: PathBuf, + limit: u64, + entries: u64, + name_bytes: u64, +} + +static OBSERVATION: Mutex> = Mutex::new(None); + +// Only the selected synthetic disk is observed; concurrent unrelated scanners +// do not consume its budget. This hook is absent from non-test builds. +pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) { + let mut guard = OBSERVATION.lock().expect("enumeration observation lock"); + if let Some(observation) = guard.as_mut() + && Path::new(dir).starts_with(&observation.root) + { + observation.entries += 1; + observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name"); + if observation.entries >= observation.limit { + budget.cancel_for_runtime(); + } + } +} + +struct ObservationGuard; + +impl Drop for ObservationGuard { + fn drop(&mut self) { + *OBSERVATION.lock().expect("enumeration observation cleanup") = None; + } +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + workspace: PathBuf, + objects: usize, + raw_entry_budget: u64, + round: u32, +} + +async fn read_bounded(path: &Path) -> Vec { + let file = tokio::fs::File::open(path).await.expect("open fixture artifact"); + let mut bytes = Vec::new(); + file.take(MAX_CACHE_BYTES + 1) + .read_to_end(&mut bytes) + .await + .expect("read fixture artifact"); + assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES); + bytes +} + +async fn round(request: &Request) -> serde_json::Value { + assert!((1..=1024).contains(&request.objects)); + assert!((1..=4096).contains(&request.raw_entry_budget)); + assert!(request.round < 64); + let disk_root = request.workspace.join("disk"); + let cache_path = request.workspace.join("cache.bin"); + if request.round == 0 { + tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk"); + for index in 0..request.objects { + let object = format!("object-{index:04}"); + let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1); + let bytes = metadata_for_object_version("bucket", &object, Some(version)); + write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await; + } + let mut initial = DataUsageCache::default(); + initial.info.name = "bucket".to_string(); + initial.info.skip_healing = true; + initial.info.snapshot_complete = false; + initial.replace("bucket", "", DataUsageEntry::default()); + tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec")) + .await + .expect("persist initial cache"); + } + let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan"); + assert_eq!(cache.info.name, "bucket"); + let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects; + let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open synthetic disk in this process"); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + *OBSERVATION.lock().expect("install observation") = Some(Observation { + root: disk.path(), + limit: request.raw_entry_budget, + entries: 0, + name_bytes: 0, + }); + let _observation_guard = ObservationGuard; + let result = scan_data_folder( + budget.token(), + budget.clone(), + vec![disk.clone()], + disk, + cache.clone(), + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + let (returned, outcome) = match result { + Ok(cache) => (cache, "complete"), + Err(ScannerError::PartialCache(cache)) => (*cache, "partial"), + Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => { + (cache, "cancelled_without_cache") + } + Err(error) => panic!("unexpected real scanner failure: {error}"), + }; + let encoded = returned.marshal_msg().expect("returned cache codec"); + assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES); + tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache"); + let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec"); + let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root"); + let scanned = returned.checked_flatten("bucket").expect("returned bucket root"); + assert_eq!( + (retained.objects, retained.versions, retained.size), + (scanned.objects, scanned.versions, scanned.size) + ); + assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete); + let guard = OBSERVATION.lock().expect("read observation"); + let observation = guard.as_ref().expect("installed observation"); + serde_json::json!({ + "schema": 1, "pid": std::process::id(), "round": request.round, + "objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget, + "raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes, + "objects_processed": budget.progress().0, + "objects_before": before, "objects_retained": retained.objects, + "versions_retained": retained.versions, "bytes_retained": retained.size, + "snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome, + }) +} + +/// Default CI is a positive healthy control. The external driver selects the +/// same worker in a fresh OS process per round and applies its strict oracle. +#[tokio::test] +#[serial] +async fn enumeration_restart_worker() { + if let Some(path) = std::env::var_os(REQUEST_ENV) { + let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request"); + let report = round(&request).await; + tokio::fs::write( + request.workspace.join(format!("round-{}.json", request.round)), + serde_json::to_vec(&report).expect("report JSON"), + ) + .await + .expect("write worker report"); + } else { + let temp = tempfile::tempdir().expect("healthy fixture directory"); + let report = round(&Request { + workspace: temp.path().to_path_buf(), + objects: 4, + raw_entry_budget: 16, + round: 0, + }) + .await; + assert_eq!(report["outcome"], "complete"); + assert_eq!(report["snapshot_complete"], true); + assert_eq!(report["objects_retained"], 4); + assert_eq!(report["versions_retained"], 4); + assert_eq!(report["bytes_retained"], 4); + assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}"); + } +} diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 19e0fc457..9521fc2fa 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -124,6 +124,16 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc) { (temp_dir, store) } +async fn wait_for_namespace_commit_tails(store: &ECStore) { + tokio::time::timeout(Duration::from_secs(30), async { + while store.scanner_data_usage_publication_blocked().await { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("namespace commit tails should drain before the scanner fixture runs"); +} + #[tokio::test] #[serial] async fn checkpoint_fixture_bucket_identity_uses_its_set_instance_owner() { @@ -334,6 +344,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() .await .expect("initial object should persist"); } + wait_for_namespace_commit_tails(store.as_ref()).await; let mut baseline = None; for (index, (scan_mode, requires_full_scan, explicit_scope)) in [ (HealScanMode::Normal, true, false), @@ -352,6 +363,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() .put_object("cold-bucket", &format!("added-{index}"), &mut reader, &ScannerObjectOptions::default()) .await .expect("cold bucket mutation should persist"); + wait_for_namespace_commit_tails(store.as_ref()).await; // Only the hot bucket is in the usage hint. The cold result must // come from this cycle's storage walk, not its previous baseline. record_dirty_usage_bucket("hot-bucket"); diff --git a/docs/operations/bucket-metadata-recovery.md b/docs/operations/bucket-metadata-recovery.md index f467ceeb1..53361a4c2 100644 --- a/docs/operations/bucket-metadata-recovery.md +++ b/docs/operations/bucket-metadata-recovery.md @@ -1,24 +1,28 @@ # Bucket metadata diagnostics and recovery -`GET /rustfs/admin/v3/export-bucket-metadata` exports every configuration it can read. A configuration that is stored but unreadable is never exported and never replaced by a fabricated default; instead the bucket gains an entry `/rustfs-unreadable-configs.json` of the shape `{"bucket": …, "unreadable": [{"config": …, "error": …}]}` naming each configuration that could not be read and why, and the export continues, so one bucket's undecodable payload cannot cost an operator the whole-cluster backup (rustfs/backlog#2309). The marker name is outside the configuration namespace importers dispatch on, so importing the archive back leaves the affected bucket's stored bytes untouched. A failure of the server's own serialization or archive writing still fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets. +`GET /rustfs/admin/v3/export-bucket-metadata` exports the supported bucket configurations as a backup. If any selected configuration is unreadable or changes between validation and export, the request fails instead of returning an archive with missing settings. Serialization and archive-writing failures also fail the request. The optional `bucket` query selects one bucket; omitting it selects all buckets. To collect a shareable support artifact that identifies the failures without carrying parser detail, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has: - Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`. - Readable entries under `_diagnostic//`; target credentials remain redacted. -- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details, and therefore carries no `rustfs-unreadable-configs.json` marker: the manifest reports the same failures with less detail, which is what makes a diagnostic archive safe to hand out. +- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details. `complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations. +Earlier partial exports containing `/rustfs-unreadable-configs.json` are also rejected before any import changes. Their omitted settings could otherwise silently disappear when restoring into an empty cluster. Preserve these archives for diagnosis, repair the source configuration, and obtain a successful ordinary export before treating it as a backup. + ## Recover unreadable replication targets RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support. -1. Inspect the diagnostic manifest, or the `rustfs-unreadable-configs.json` marker in an ordinary export, to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery. +1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery. 2. Prepare a ZIP containing `/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets. 3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully. -4. Verify target listing and the intended replication configuration. Retry the ordinary metadata export to confirm the bucket no longer carries an unreadable marker. +4. Verify target listing and the intended replication configuration. Retry the ordinary metadata export and confirm it succeeds. -Alternatively, `PUT /rustfs/admin/v3/set-remote-target?replace-unreadable=true` discards an undecodable target set as part of setting a replacement target. The flag is the operator's explicit acknowledgement that the stored set is being thrown away; without it the request is refused rather than rewriting an unreadable set from a partial view. +Alternatively, use `PUT /rustfs/admin/v3/set-remote-target?bucket=&replace-unreadable=true` with a complete target-create payload, including the endpoint, target bucket, target type, and credentials. This mode cannot be combined with `update=true`. Both `update` and `replace-unreadable` accept only a single `true` or `false` value; duplicate parameters and other values are rejected before any configuration write. + +RustFS validates the replacement target, then reads and updates the latest persisted target set under the bucket metadata transaction lock. The flag authorizes discarding that set only if it is still unreadable at this point. A readable set, including a repair already committed by another node, is preserved and merged using the ordinary target-create identity and conflict checks. Success is returned after persistence, and an actual discard is audited after the commit. Without this opt-in, target writes retain the unreadable-configuration refusal. Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive. diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index cedfc4966..04f06aee4 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -85,6 +85,13 @@ at most 1 MiB. Logs are kept separately and require an operator-managed disk quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero samples, and request errors fail the run. Adapters must terminate their own children on failure and `stop` must be idempotent even after partial preparation. +The runner keeps its session leader unreaped while stopping a failed command +or collector: it sends TERM, allows the existing ten-second grace period, then +kills the remaining process group before reaping. This prevents a parent exit +from hiding live descendants or allowing the group ID to be reused before its +last signal. A successful `prepare` preserves adapter-owned services until +`stop`; services that leave the command's process group remain the adapter's +cleanup responsibility. The request contains the fixed manifest fields, selected build, scenario, round, leg, comparison (`build` or `background`), background mode (`on` or `off`), diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 1779fe458..b40f1ee5e 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -121,8 +121,113 @@ Update this file in the same PR when a job or check name changes, a workflow gai The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact. -The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. +The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, corrupt part arrays, and the shared on-demand-migration source-backend contract for each provider dialect (S3, Azure, native GCS). The three contract entries live in the `rustfs` suite and reach the lane through that package's default features, so dropping `gcs` from `rustfs`'s defaults fails this check instead of silently deselecting the GCS contract (rustfs/backlog#2323). Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability. Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review. +## Scanner/Heal Evidence Receipts + +The existing `scripts/check_test_wiring.py` also validates Scanner/Heal case +evidence registered in `.config/scanner-heal-required-tests.json`. It records +already-built binaries and checks existing nextest output; it does not build, +run tests, deploy servers, inject faults, or start another CI lane. + +The initial case is `background-target-restart`, emitted by +`heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`. +That test already runs in `e2e-nightly`. When `RUSTFS_SCANNER_HEAL_RUN_DIR` is set, +it checks the actual server and test-executable hashes against `run.json`, pins +the same server binary for all node starts, and writes its oracle only after +the real assertions pass. The artifact contains the actual pre/post target +PIDs, per-node S3 listings, expected and downloaded complete-body hashes/lengths, +and target-disk `VersionShardCensus` fingerprints. Existing baseline objects +must match their pre-fault physical manifests; the object created during the +outage has no pre-fault target shard and is checked for complete physical parts +and exact S3 content. + +This case is a **four-node, one-drive-per-node process-restart test**. It is not +power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof +of scanner enumeration, exact MRF disposition, legacy migration, or rollback. +The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending +until their actual feature-specific oracles and required topologies exist. +Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON +and file-hash helpers are reused; its ABBA performance contracts remain in +`docs/operations/scanner-benchmark-runbook.md`. + +### Recording One Case + +Use a committed source tree, independently built current binaries, sufficient +free disk space, and a task-owned artifact directory that does not yet exist. +Set `SERVER_BINARY` and `TEST_BINARY` to those exact executable paths. The begin +command requires the server's embedded `--version` commit to match the clean +checkout and its embedded Git status to be clean. The E2E crate's build script +embeds its build-time Git revision/dirty state, lockfile Git blob, enabled crate +features, target, profile and encoded Rust flags. It tracks the crate/dependency +trees, Cargo inputs and Git HEAD/ref/index, including `common.rs` restart logic. +The producer checks this compiled identity against the receipt; it does not +copy a current source revision into an older test binary's identity. The E2E +uses its existing temporary cluster directories and cleanup. `CARGO_TARGET_DIR` +controls compilation output; nextest's default report store remains the +workspace's `target/nextest`. Execute the existing selected case as follows: + +```bash +CASE=background-target-restart +FILTER='test(test_cluster_root_heal_recovers_remote_shards_after_background_target_restart)' +RUN_DIR="$PWD/artifacts/scanner-heal-run" +export RUSTFS_E2E_EXPECTED_FEATURES=default +scripts/python_bin.sh scripts/check_test_wiring.py \ + --begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY" +export RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" +export CARGO_BIN_EXE_rustfs="$SERVER_BINARY" +cargo nextest list --profile e2e-nightly -p e2e_test -E "$FILTER" \ + --message-format json > "$RUN_DIR/listing.json" +rm -f target/nextest/e2e-nightly/junit.xml +set +e +cargo nextest run --profile e2e-nightly -p e2e_test -E "$FILTER" +test_exit=$? +set -e +cp target/nextest/e2e-nightly/junit.xml "$RUN_DIR/junit.xml" +scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_DIR" "$test_exit" +scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE" +``` + +Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set, +including `default` for a default-feature build, comma-separated for extra +features, or empty for `--no-default-features`. It is mandatory when beginning +a run. Crate features are distinct from the spawned server's build features. + +Do not replace a nonzero command exit with zero. Missing JUnit or an oracle +emission failure also fails acceptance. Each retry needs a new run directory; +the producer refuses to overwrite an existing oracle. Keep failed-run logs and +artifacts. The receipt pins source revision, actual binary hashes, run identity, +start/finish times, and the artifact hashes. `listing.json`, `junit.xml`, and +each oracle are limited to 1 MiB; object evidence has the fixture's 9..65 object +bound. Credentials are not included in the receipt. + +The checker binds nextest's flattened suite `binary-id`/`binary-path` to the +actual test executable and requires the JUnit testcase's embedded execution +timestamp to fall inside the receipt window (with millisecond precision). +Copying an old JUnit file and refreshing its mtime does not make it new evidence. +Schema versions, topology counts, PIDs, EC geometry and shard indices require +actual integers: booleans and fractional values are rejected, and an index must +fit the physical data-plus-parity geometry. + +The checker rejects unselected/ignored tests, zero/duplicate JUnit cases, +failures, skipped tests, retry/flaky records, stale or changed artifacts, +different builds or run IDs, unchanged process IDs, wrong topology, missing +shard parts, and mismatched S3 content/listings. The raw oracle JSON is emitted +by the real E2E producer, not accepted from an adapter copying expectations. + +`--check-scanner-heal "$RUN_DIR" release` checks available case evidence and +returns nonzero for every pending release requirement. A focused case pass +does not approve release. In particular, R-E requires fixed-budget real +restarts without an unbudgeted final sweep, R-D requires the full +manager/event/ledger disposition chain, and R-L requires source-conflict and +crash/retirement evidence. Reader-only or unit fixtures cannot substitute for +these. The external `rustfs/auto-testing` functional workflows propagate suite +failures. Their workflow status does not establish this registry's required +case coverage, build provenance, or object-level oracles. + +Run parser/receipt regressions with +`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures +validate the checker only and produce no runtime or performance evidence. diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index d8e3ec2bc..10d882f9b 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -17,7 +17,7 @@ A multi-pool layout in which any pool spans several localhost ports is not expre Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran. -The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated ext4 loopback filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. +The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing. @@ -58,6 +58,11 @@ Hardware power-loss, physical NIC pull, authenticated inter-node partition, firm ```bash cargo build -p rustfs --bins # Expansion/decommission/rebalance cases require four paths on distinct filesystems. +# If you do not already have four disks, sized tmpfs is enough: +# for p in 0 1 2 3; do +# sudo mkdir -p /mnt/rustfs-pool-$p +# sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs /mnt/rustfs-pool-$p +# done export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3 # Upgrade cases require the pinned previous binary (CI downloads it). export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2 diff --git a/docs/testing/mrf-pending-migration.md b/docs/testing/mrf-pending-migration.md new file mode 100644 index 000000000..f07638c1c --- /dev/null +++ b/docs/testing/mrf-pending-migration.md @@ -0,0 +1,21 @@ +# Pending MRF Migration + +`heal::mrf_queue::snapshot::migration` exposes explicit capture, staging, and readback of pending legacy responsibility evidence. Nothing invokes it from the production MRF consumer. It does not enable the committed-snapshot writer, freeze legacy ingress, acknowledge durable admission, or authorize source garbage collection. + +The caller supplies every configured local disk slot, including missing slots. Missing/unformatted/duplicate disks, unavailable metadata volumes, invalid records, and aggregate byte/record/source-history overflow fail closed. A valid empty source observation can inherit earlier pending responsibilities, but staging without any current or inherited responsibility is rejected. Both legacy paths retain their original bytes, disk identity, absent-versus-empty state, and SHA-256 digest. Complete subset/superset replicas become conservative pending evidence, never a claimed newest legacy snapshot. Raw record replay preserves kind, scope and nil/absent version encodings; unknown incarnation stays unknown. + +Staging writes only `.heal-mrf-import-pending.{0,1}.bin`, `.heal-mrf-import-commit.{0,1}.bin`, and `.heal-mrf-import-claim.bin` under the metadata volume. It reuses the committed reader's manifest codec and the storage owner's conditional-file operation, including the configured metadata durability policy. A candidate is written before sources are revalidated, its manifest is then committed, and committed bytes plus source coverage are read back before success. Success is pending staging evidence, not a power-loss or cluster-quorum durability receipt. + +A successor inherits prior source bytes even if a replay consumer has already read or admitted their records. Independent byte, record, and source-history limits include inherited evidence, and source identities use a hash index with full-byte conflict checks. There is no completion-based pruning. A changed source blocks recovery of that pending generation; an explicit new capture can stage a successor that retains both the previous and current responsibilities. The inactive slot is replaced while the preceding committed slot remains intact. Any corrupt, unsupported or over-budget slot blocks selection rather than falling back to an older generation. The reader first collects bounded lineage evidence from at most two slots per configured disk. A payload without a manifest is repairable only when its length and digest match the complete retry candidate after inheritance, or any independently validated committed payload, including an older generation on another replica. Unknown payloads still block writing. Retries also repair missing replica manifests. + +All participating disks are claimed in disk-identity order through CAS. Normal completion conditionally releases only the current invocation's claim, attempts every acquired claim even after a release error, and reports the first release error. Cancellation, process death, or an ambiguous claim/release I/O failure may leave a claim behind. Read-only recovery remains available when the snapshot/source proof is valid, but further staging is blocked until a separate storage-fenced recovery procedure is implemented. Process liveness and the legacy ingress lease do not authorize taking over or deleting a claim. + +Run the focused fixtures with a nonzero test count: + +```sh +cargo test -p rustfs-heal --lib heal::mrf_queue::snapshot::migration::tests +``` + +Fixtures use real local disks and the production CAS/readback path. They cover disk-order independence, retained raw identities, source change after candidate write, capacity rejection, interrupted commit boundaries, lost responses, torn inactive slots, and refusal to take over an interrupted claim. Boundary injection and same-process reopen are not process-kill, directory-fsync failure, disk-full, mixed-version, or power-loss tests. The actual manager Full/Accepted-to-crash pipeline remains outside this staged API. + +Rollback leaves all pending and legacy artifacts untouched. Activation still requires legacy-writer coordination, bounded recoverable ingress, exact object-disposition/successor receipts, and the W14/W21 process-crash and compatibility gates. The production legacy replay deletion window remains unresolved by this staging-only phase. diff --git a/docs/testing/scanner-checkpoint-fixture.md b/docs/testing/scanner-checkpoint-fixture.md index 1c413a4b3..f9d437905 100644 --- a/docs/testing/scanner-checkpoint-fixture.md +++ b/docs/testing/scanner-checkpoint-fixture.md @@ -1,5 +1,38 @@ # Scanner Checkpoint Fixture +## Raw Enumeration Restart Diagnostic + +`enumeration_restart_worker` exercises the real `scan_data_folder` with a local disk and valid `xl.meta` objects. Without configuration it is a positive CI control: four one-byte objects must complete and survive a cache codec round trip. It is not an ignored test or an assertion that a known defect must persist. + +```sh +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib enumeration_restart_worker -- --nocapture +cargo test -p rustfs-scanner --lib --no-run --message-format=json +python3 -m unittest discover -s scripts -p 'test_diagnose_scanner_enumeration_restart.py' +``` + +Use the `executable` from the scanner library test `compiler-artifact` JSON record as `--test-binary` below. The driver verifies that it contains the exact worker test before doing any work; a zero-test filter cannot pass. + +```sh +python3 scripts/diagnose_scanner_enumeration_restart.py \ + --test-binary /path/to/compiled/scanner-libtest \ + --output /tmp/scanner-enumeration-new-run \ + --objects 128 --raw-entry-budget 8 --rounds 8 +``` + +The output directory must not exist. Each round starts a new OS test-worker process, opens the same synthetic disk, decodes the preceding cache, invokes the real scanner, encodes the returned cache, and decodes it again. When cancellation returns no useful partial cache, it preserves the previous cache. Reports identify the actual child PID, round, raw entries and name bytes observed, processed objects, retained object/version/byte counts, and completeness. No observed-name set, `readdir` offset, or assumed stable ordering is used as durable progress. Namespace creation happens only during fixture setup, before scan accounting. + +The `cfg(test)` hook observes actual entries delivered by `read_dir` and cancels the existing cycle token at the fixed entry limit. This is a deterministic injected **raw-entry work budget**, not a wall-clock performance measurement or a claim that kernel prefetch, probes, allocations, name bytes, or cache I/O are independently budgeted. The watchdog timeout only bounds worker lifetime. The hook does not replace enumeration, classification, or recursion, and does not exist in production builds. In particular, `xl.meta` object-boundary classification is unchanged. + +Exit 0 requires exact complete object/version/byte coverage within the same fixed budget on every executed round. Exit 1 means the strict convergence oracle remains unmet, including the current flat-directory enumeration starvation case. Exit 2 means invalid input, worker failure, or invalid evidence; it is not a successful reproduction. There is no final unbudgeted sweep. Small fixtures can pass; that does not establish the general R-E gate from [the scanner review comment](https://github.com/rustfs/backlog/issues/2240#issuecomment-5549222480). Raw entries observed are not a retained enumeration watermark. This is scanner-worker process restart plus codec evidence, **not** whole-daemon restart, EC quorum persistence, crash/fsync durability, remote RPC, or a throughput benchmark. The caller owns the bounded evidence directory and may remove it after inspection. + +### Missing Storage Capability + +The current `scanner_folder::FolderScanner::scan_folder` collects child folders before recursing. `LocalDisk::scan_dir` also reads the whole parent before sorting and applying `forward_to`. The persistent key-only listing index's `collect_persistent_key_only_index_objects` / `rebuild_persistent_key_only_index` collects all objects in memory before publication and excludes deleted entries. It cannot supply a restartable first-build cursor over per-disk raw entries, orphan directories, and metadata boundaries. Repeated listing from the beginning is real work, not free pagination. + +A future storage-owner capability must expose an explicit unsupported/building/ready state and a durable snapshot/index identity bound to disk mount, bucket incarnation, and directory identity. It must budget the first build and every page, including entry count, name bytes, metadata probes, I/O and time; survive a process restart during first build; seal page data before advancing the manifest; and distinguish enumerated, classified, and fully processed frontiers. An uncommitted page may be replayed only within a bounded cost. `xl.meta` classification must finish before descendants become traversable namespace. Missing capability or invalid identities must not become fabricated progress or completeness. No such capability is implemented by this diagnostic, and ordinary local storage remains without this R-E guarantee. + +## Completed Subtree Checkpoint Fixture + The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark. Run the fixture and confirm the test filter selects a nonzero number of tests: @@ -24,3 +57,19 @@ This fixture bounds object processing after directory enumeration. It does not p For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store. The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Rolling back to a reader without the optional checkpoint metadata rebuilds partial coverage; it must not clear quota floors or complete authoritative snapshots. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches. + +## Segment Observation Diagnostics + +The nested `segment_observation` fixture compares diagnostic on/off runs of the real folder walker over six objects in `hot/`, `cold/`, and `other/`. Each run first rewrites `hot/one` with a different, equal-length ETag in real fixture metadata and reads it back to verify changed bytes at unchanged length. The successful fixture write supplies its known key to a diagnostic executed inside the real walker's path callback. Both runs save and reload the actual cache through the existing codec and revision-aware file backend. Assertions compare traversal order and the entire decoded cache, not encoded map order or aggregate size alone. Proposed top-level segments never reach a scanner selector or publication decision, and non-proposed segments must still be walked. The diagnostic retains at most four segments and 128 segment-name bytes; actual-walk samples are limited to 32 entries and 1,024 bytes. Exceeding sample limits fails the fixture rather than silently truncating its oracle. Saving a cache here is not an authoritative root publication. + +Entry/byte overflow and malformed keys reject the fixture proposal. Missing producers, process restarts, event gaps, and compacted child coverage remain **unverified production capabilities**, not simulated success cases in this fixture. Mainline bucket dirty generations and hashed metadata-cache invalidation stripes are not an exact, replayable object-key stream. The open [prefix reuse proposal #7208](https://github.com/rustfs/rustfs/pull/7208) is a separate candidate implementation; these tests neither import its hint map nor activate its skip path. + +The ECStore `segment_observation_equal_size_mutations_retire_metadata_generation` test uses the existing exact-key, test-only invalidation probe and actual owner operations. A same-length PUT must change the returned body and ETag while retiring the old generation; metadata-only PUT must change returned metadata and retire the old generation while size and ETag remain equal. Setup uses the existing full-fanout cache-priming helper; the observed mutations use normal owner locking. This is focused producer evidence, not an end-to-end connection between the owner probe and scanner range selection. The existing semantic mutation matrix covers additional owner entry points separately. + +```sh +cargo test -p rustfs-scanner --lib segment_observation -- --list +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib segment_observation -- --nocapture +RUST_MIN_STACK=4194304 cargo test -p rustfs-ecstore --lib segment_observation_equal_size_mutations_retire_metadata_generation -- --nocapture +``` + +[W19](https://github.com/rustfs/backlog/issues/2272) remains open for trustworthy producer coverage, source/incarnation binding, and production shadow observations. No production stream, durable journal, runtime feature switch, scan skipping, or performance claim is introduced here. No restart/gap detection or restart-safe production coverage is established, and the revision-aware file backend does not prove EC publication durability. diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 396e66187..e6a8ef9bb 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -67,15 +67,8 @@ use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic"; const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json"; -/// Archive entry naming the configurations a bucket stores but this build -/// could not read (rustfs/backlog#2309). -/// -/// The name deliberately sits outside the configuration-file namespace the -/// importers switch on — `ImportBucketMetadata` matches known configuration -/// names and ignores everything else — so no importer can mistake the marker -/// for a configuration. Ordinary exports carry it; diagnostic exports report -/// the same failures through [`DIAGNOSTIC_EXPORT_MANIFEST`] instead, which -/// deliberately withholds the parser detail this marker records. +/// Earlier partial exports omitted configurations and carried this marker. +/// Reject those archives before import can create an incomplete replacement. const EXPORT_UNREADABLE_MANIFEST: &str = "rustfs-unreadable-configs.json"; const LOG_COMPONENT_ADMIN: &str = "admin"; @@ -87,32 +80,15 @@ fn export_internal_error(message: impl Into) -> s3s::S3Error { s3_error!(InternalError, "{message}") } -/// One configuration that is stored for a bucket but could not be exported. -#[derive(serde::Serialize)] -struct UnreadableExportEntry { - config: &'static str, - error: String, -} - -#[derive(serde::Serialize)] -struct UnreadableExportManifest<'a> { - bucket: &'a str, - unreadable: &'a [UnreadableExportEntry], -} - /// Why one of a bucket's configurations could not be exported. /// -/// The two variants are what an ordinary export dispatches on: a bucket whose -/// stored bytes this build cannot turn into a configuration is named and -/// skipped, while a failure of our own output machinery still fails the whole -/// export closed. +/// Ordinary backups fail on either variant. Explicit diagnostics may report +/// unreadable configurations, but output failures still abort the archive. #[derive(Debug)] enum ExportConfigError { /// The configuration is stored but this build cannot read it: a /// MinIO-origin or otherwise undecodable blob, or a revision that moved - /// underneath the export. No retry of ours turns those bytes into a - /// configuration, so one such bucket must not abort a whole-cluster export - /// (rustfs/backlog#2309). + /// underneath the export. Only an explicit diagnostic export may omit it. Unreadable(String), /// This build failed to produce its own output for a configuration it had /// already decoded. Nothing about the stored bytes is in doubt, so the @@ -179,62 +155,50 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result { - let config: s3s::dto::NotificationConfiguration = match metadata_sys::get_notification_config(bucket).await { - Ok(Some(res)) => res, - Err(e) => { - if e == StorageError::ConfigNotFound { - return Ok(None); - } - return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}"))); - } - Ok(None) => return Ok(None), - }; - - let raw_config = metadata_sys::get(bucket) + let metadata = metadata_sys::get(bucket) .await - .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))? - .notification_config_xml - .clone(); - let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; - + .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?; + if metadata.notification_config_xml.is_empty() { + return Ok(None); + } + let config = metadata + .notification_config + .as_ref() + .ok_or_else(|| ExportConfigError::unreadable("persisted bucket notification configuration is invalid"))?; + let config_xml = checked_raw_xml( + config, + metadata.notification_config_xml.clone(), + deserialize::, + )?; Ok(Some(config_xml)) } BUCKET_LIFECYCLE_CONFIG => { - let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(bucket).await { - Ok((res, _)) => res, - Err(e) => { - if e == StorageError::ConfigNotFound { - return Ok(None); - } - return Err(ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}"))); - } - }; - let raw_config = metadata_sys::get(bucket) + let metadata = metadata_sys::get(bucket) .await - .map_err(|e| ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")))? - .lifecycle_config_xml - .clone(); - let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; - + .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?; + if metadata.lifecycle_config_xml.is_empty() { + return Ok(None); + } + let config = metadata + .lifecycle_config + .as_ref() + .ok_or_else(|| ExportConfigError::unreadable("persisted bucket lifecycle configuration is invalid"))?; + let config_xml = + checked_raw_xml(config, metadata.lifecycle_config_xml.clone(), deserialize::)?; Ok(Some(config_xml)) } BUCKET_TAGGING_CONFIG => { - let config: Tagging = match metadata_sys::get_tagging_config(bucket).await { - Ok((res, _)) => res, - Err(e) => { - if e == StorageError::ConfigNotFound { - return Ok(None); - } - return Err(ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}"))); - } - }; - let raw_config = metadata_sys::get(bucket) + let metadata = metadata_sys::get(bucket) .await - .map_err(|e| ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")))? - .tagging_config_xml - .clone(); - let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; - + .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?; + if metadata.tagging_config_xml.is_empty() { + return Ok(None); + } + let config = metadata + .tagging_config + .as_ref() + .ok_or_else(|| ExportConfigError::unreadable("persisted bucket tagging configuration is invalid"))?; + let config_xml = checked_raw_xml(config, metadata.tagging_config_xml.clone(), deserialize::)?; Ok(Some(config_xml)) } BUCKET_QUOTA_CONFIG_FILE => { @@ -439,53 +403,25 @@ impl Operation for ExportBucketMetadata { ]; for bucket in buckets { - let mut unreadable: Vec = Vec::new(); for &conf in confs.iter() { let conf_path = path_join_buf(&[bucket.name.as_str(), conf]); let config = match exported_bucket_config(&bucket.name, conf).await { Ok(Some(config)) => config, Ok(None) => continue, - Err(error) => { - if query.diagnostic { - // A diagnostic archive names every configuration it - // could not export under one fixed code, carrying - // neither the payload nor the parser detail, so it - // stays shareable (rustfs/rustfs#7225). - errors.push(serde_json::json!({ - "bucket": bucket.name, - "config": conf, - "code": "configuration_unavailable", - })); - continue; - } - match error { - // One bucket's undecodable blob must not abort the - // whole-cluster export: record which configuration - // could not be read and keep going, so an operator - // migrating away still gets every readable - // configuration (rustfs/backlog#2309). - ExportConfigError::Unreadable(error) => { - warn!( - event = EVENT_ADMIN_BUCKET_META_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_BUCKET_META, - action = "export_bucket_metadata", - result = "config_unreadable", - bucket = %bucket.name, - config_name = %conf, - error = %error, - "admin bucket meta state" - ); - unreadable.push(UnreadableExportEntry { config: conf, error }); - continue; - } - // Our own encoder failed on a configuration this - // build had already decoded. The stored bytes are - // not in question, so fail the export instead of - // reporting readable metadata as unreadable. - ExportConfigError::Internal(error) => return Err(error), + Err(ExportConfigError::Unreadable(error)) => { + if !query.diagnostic { + return Err(export_internal_error(format!("failed to export {conf_path}: {error}"))); } + // Diagnostics identify omitted configurations without + // exposing payloads or parser details. + errors.push(serde_json::json!({ + "bucket": bucket.name, + "config": conf, + "code": "configuration_unavailable", + })); + continue; } + Err(ExportConfigError::Internal(error)) => return Err(error), }; let conf_path = if query.diagnostic { path_join_buf(&[DIAGNOSTIC_EXPORT_PREFIX, &conf_path]) @@ -499,23 +435,6 @@ impl Operation for ExportBucketMetadata { .write_all(&config) .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; } - - // Only reachable outside diagnostic mode, which reports the same - // failures through the archive-wide manifest instead. - if !unreadable.is_empty() { - let manifest = serde_json::to_vec(&UnreadableExportManifest { - bucket: bucket.name.as_str(), - unreadable: &unreadable, - }) - .map_err(|e| export_internal_error(format!("failed to serialize unreadable manifest: {e}")))?; - let manifest_path = path_join_buf(&[bucket.name.as_str(), EXPORT_UNREADABLE_MANIFEST]); - zip_writer - .start_file(manifest_path, SimpleFileOptions::default()) - .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?; - zip_writer - .write_all(&manifest) - .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; - } } if query.diagnostic { @@ -626,8 +545,12 @@ impl Operation for ImportBucketMetadata { || path .strip_prefix(DIAGNOSTIC_EXPORT_PREFIX) .is_some_and(|suffix| suffix.starts_with('/')) + || path.rsplit('/').next() == Some(EXPORT_UNREADABLE_MANIFEST) }) { - return Err(s3_error!(InvalidRequest, "diagnostic bucket metadata archives cannot be imported")); + return Err(s3_error!( + InvalidRequest, + "diagnostic or incomplete bucket metadata archives cannot be imported" + )); } let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?; @@ -1554,6 +1477,137 @@ mod backup_zip_compatibility_tests { assert_eq!(response.output.0, StatusCode::OK); } + async fn assert_unreadable_xml_export_is_explicit(config_file: &str) { + const RAW_SECRET: &[u8] = b"unreadable-xml-with-private-config"; + let _ = rustfs_credentials::init_global_action_credentials( + Some(ROOT_ACCESS_KEY.to_string()), + Some(ROOT_SECRET_KEY.to_string()), + ); + let temp = tempfile::tempdir().expect("create unreadable XML export test root"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp.path()) + .disk_count(1) + .build() + .await; + env.make_bucket(BUCKET, false).await; + rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore)) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)) + .await + .expect("build test IAM"); + publish_test_app_context(Arc::new(AppContext::with_default_interfaces( + Arc::clone(&env.ecstore), + iam, + Arc::new(rustfs_kms::KmsServiceManager::new()), + ))); + metadata_sys::update(BUCKET, BUCKET_VERSIONING_CONFIG, VERSIONING_XML.to_vec()) + .await + .expect("persist readable companion config"); + assert!( + exported_bucket_config(BUCKET, config_file) + .await + .expect("absent configuration") + .is_none() + ); + + let mut metadata = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("load source metadata"); + match config_file { + BUCKET_NOTIFICATION_CONFIG => metadata.notification_config_xml = RAW_SECRET.to_vec(), + BUCKET_LIFECYCLE_CONFIG => metadata.lifecycle_config_xml = RAW_SECRET.to_vec(), + BUCKET_TAGGING_CONFIG => metadata.tagging_config_xml = RAW_SECRET.to_vec(), + _ => panic!("unexpected unreadable XML fixture"), + } + metadata + .save_with_store(Arc::clone(&env.ecstore)) + .await + .expect("persist raw configuration with a failed parse"); + crate::storage::storage_api::set_bucket_metadata(BUCKET.to_string(), metadata) + .await + .expect("publish unreadable XML fixture"); + + let ordinary = ExportBucketMetadata {} + .call( + admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()), + Params::new(), + ) + .await + .expect_err("persisted invalid XML must not disappear from an ordinary backup"); + assert_eq!(ordinary.code(), &s3s::S3ErrorCode::InternalError); + + let diagnostic = ExportBucketMetadata {} + .call( + admin_request( + Method::GET, + Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"), + Vec::new(), + ), + Params::new(), + ) + .await + .expect("diagnostic export must identify the omitted configuration"); + assert_eq!(diagnostic.output.0, StatusCode::OK); + let bytes = diagnostic + .output + .1 + .collect() + .await + .expect("read diagnostic archive") + .to_bytes(); + let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open diagnostic archive"); + let mut files = HashMap::new(); + for index in 0..archive.len() { + let mut file = archive.by_index(index).expect("read diagnostic entry"); + let mut content = Vec::new(); + file.read_to_end(&mut content).expect("read diagnostic config"); + assert!(!content.windows(RAW_SECRET.len()).any(|window| window == RAW_SECRET)); + files.insert(file.name().to_string(), content); + } + assert!(!files.contains_key(&format!("_diagnostic/{BUCKET}/{config_file}"))); + assert_eq!(files[&format!("_diagnostic/{BUCKET}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML); + let manifest: serde_json::Value = + serde_json::from_slice(&files[DIAGNOSTIC_EXPORT_MANIFEST]).expect("decode diagnostic manifest"); + assert_eq!( + manifest, + serde_json::json!({ + "version": 1, + "mode": "diagnostic", + "complete": false, + "errors": [{ "bucket": BUCKET, "config": config_file, "code": "configuration_unavailable" }], + }) + ); + assert_eq!( + persisted_xml( + &metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read unchanged source"), + config_file + ), + RAW_SECRET + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn unreadable_notification_xml_fails_backup_and_is_named_in_diagnostics() { + assert_unreadable_xml_export_is_explicit(BUCKET_NOTIFICATION_CONFIG).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn unreadable_lifecycle_xml_fails_backup_and_is_named_in_diagnostics() { + assert_unreadable_xml_export_is_explicit(BUCKET_LIFECYCLE_CONFIG).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn unreadable_tagging_xml_fails_backup_and_is_named_in_diagnostics() { + assert_unreadable_xml_export_is_explicit(BUCKET_TAGGING_CONFIG).await; + } + #[tokio::test] #[serial_test::serial] async fn diagnostic_export_isolated_errors_and_import_recovers_unreadable_targets() { @@ -1611,62 +1665,18 @@ mod backup_zip_compatibility_tests { .expect("publish unreadable targets fixture"); assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err()); - // rustfs/backlog#2309: an ordinary export no longer fails closed on - // a configuration that is stored but unreadable. It names that one - // configuration in the bucket's own marker entry and keeps every - // readable configuration of every bucket, so one MinIO-origin blob - // cannot cost an operator the whole-cluster backup. let ordinary = ExportBucketMetadata {} .call( admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()), Params::new(), ) .await - .expect("one unreadable configuration must not abort the ordinary export"); - assert_eq!(ordinary.output.0, StatusCode::OK); - assert!(!ordinary.headers.contains_key("x-rustfs-bucket-metadata-export")); - let ordinary_bytes = ordinary.output.1.collect().await.expect("read ordinary archive").to_bytes(); - let mut ordinary_archive = ZipArchive::new(Cursor::new(&ordinary_bytes)).expect("open ordinary archive"); + .expect_err("an ordinary backup must fail rather than omit an unreadable configuration"); + assert_eq!(*ordinary.code(), s3s::S3ErrorCode::InternalError); assert!( - ordinary_archive - .by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")) - .is_ok(), - "a healthy bucket must still export while another bucket is unreadable" - ); - assert!( - ordinary_archive - .by_name(&format!("{HEALTHY}/{EXPORT_UNREADABLE_MANIFEST}")) - .is_err(), - "a bucket whose configurations all read must carry no unreadable marker" - ); - assert!( - ordinary_archive - .by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")) - .is_err(), - "an unreadable targets blob must never be exported as a configuration" - ); - let mut ordinary_marker = Vec::new(); - ordinary_archive - .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}")) - .expect("the ordinary export must name the configuration it could not read") - .read_to_end(&mut ordinary_marker) - .expect("read unreadable marker"); - assert!( - !ordinary_marker - .windows(SECRET.len()) - .any(|window| window == SECRET.as_bytes()) - ); - let ordinary_marker: serde_json::Value = serde_json::from_slice(&ordinary_marker).expect("the marker must be JSON"); - assert_eq!(ordinary_marker["bucket"], UNREADABLE); - assert_eq!( - ordinary_marker["unreadable"].as_array().map(Vec::len), - Some(1), - "only the configuration that could not be read may be marked: {ordinary_marker}" - ); - assert_eq!(ordinary_marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE); - assert!( - ordinary_marker["unreadable"][0]["error"].is_string(), - "the marker must carry the reason an operator needs to repair the bucket" + ordinary + .message() + .is_some_and(|message| message.contains(BUCKET_TARGETS_FILE)) ); let response = ExportBucketMetadata {} @@ -1739,31 +1749,44 @@ mod backup_zip_compatibility_tests { ); } - // The marker may be malformed, come last, or be removed while the - // reserved directory remains. None may allow an earlier config write. + // Diagnostic and historical partial-export markers may come last. + // Neither may allow an earlier configuration write or bucket creation. for marker in [ DIAGNOSTIC_EXPORT_MANIFEST.to_string(), DIAGNOSTIC_EXPORT_PREFIX.to_string(), format!("{DIAGNOSTIC_EXPORT_PREFIX}/bucket/config"), + format!("diagnostic-never-created/{EXPORT_UNREADABLE_MANIFEST}"), ] { let mut writer = ZipWriter::new(Cursor::new(Vec::new())); writer .start_file(format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default()) - .expect("start ordinary config before diagnostic marker"); + .expect("start ordinary config before rejected marker"); writer .write_all(b"Suspended") - .expect("write ordinary config before diagnostic marker"); + .expect("write ordinary config before rejected marker"); writer .start_file( format!("diagnostic-never-created/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default(), ) - .expect("start a nonexistent bucket config before diagnostic marker"); + .expect("start a nonexistent bucket config before rejected marker"); writer.write_all(VERSIONING_XML).expect("write nonexistent bucket config"); + let marker_content = if marker.ends_with(EXPORT_UNREADABLE_MANIFEST) { + serde_json::to_vec(&serde_json::json!({ + "bucket": "diagnostic-never-created", + "unreadable": [ + { "config": BUCKET_TARGETS_FILE, "error": "targets could not be read" }, + { "config": OBJECT_LOCK_CONFIG, "error": "object lock could not be read" }, + ], + })) + .expect("encode historical partial-export marker") + } else { + b"not json".to_vec() + }; writer .start_file(marker, SimpleFileOptions::default()) - .expect("start diagnostic marker"); - writer.write_all(b"not json").expect("write malformed diagnostic marker"); + .expect("start rejected marker"); + writer.write_all(&marker_content).expect("write rejected marker"); let error = ImportBucketMetadata {} .call( admin_request( @@ -1774,7 +1797,7 @@ mod backup_zip_compatibility_tests { Params::new(), ) .await - .expect_err("diagnostic preflight must reject before any config write"); + .expect_err("non-restorable archive preflight must reject before any config write"); assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest); assert_eq!( metadata_sys::get_config_from_disk(HEALTHY) @@ -1788,7 +1811,7 @@ mod backup_zip_compatibility_tests { .get_bucket_info("diagnostic-never-created", &BucketOptions::default()) .await .is_err(), - "diagnostic preflight must reject before bucket creation" + "non-restorable archive preflight must reject before bucket creation" ); } @@ -1845,7 +1868,7 @@ mod backup_zip_compatibility_tests { archive .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}")) .is_err(), - "the marker must disappear once the configuration reads again" + "ordinary backups must never contain partial-export markers" ); } @@ -1990,18 +2013,15 @@ mod backup_zip_compatibility_tests { let _: ReplicationConfiguration = deserialize(&restored.replication_config_xml).expect("old parser must read the newly exported archive payload"); - // rustfs/backlog#2309: a MinIO-origin `.metadata.bin` stores its - // targets as a bare JSON array, which `BucketTargets` cannot decode. - // Since rustfs/rustfs#7172 that reads as "stored but unreadable" — - // which must mark one bucket's one configuration, not abort the - // whole-cluster export an operator needs to migrate away. + // The array-shaped compatibility fixture is unreadable as targets. + // A backup must not claim success after silently omitting that setting. env.make_bucket(UNREADABLE_BUCKET, false).await; metadata_sys::update(UNREADABLE_BUCKET, BUCKET_TARGETS_FILE, MINIO_ARRAY_TARGETS.to_vec()) .await - .expect("persist the MinIO-shaped targets blob"); + .expect("persist the array-shaped targets blob"); metadata_sys::get_bucket_targets_config(UNREADABLE_BUCKET) .await - .expect_err("a MinIO array-shaped targets blob must read as unreadable, not as an empty set"); + .expect_err("an array-shaped targets blob must read as unreadable, not as an empty set"); let cluster_export = ExportBucketMetadata {} .call( @@ -2009,68 +2029,14 @@ mod backup_zip_compatibility_tests { Params::new(), ) .await - .expect("one bucket's unreadable configuration must not abort the whole-cluster export"); - assert_eq!(cluster_export.output.0, StatusCode::OK); - let cluster_archive = cluster_export - .output - .1 - .collect() - .await - .expect("read cluster archive body") - .to_bytes() - .to_vec(); - let mut archive = ZipArchive::new(Cursor::new(&cluster_archive)).expect("open cluster archive"); - - // Every readable configuration of every other bucket still exports. - for (config_file, payload) in persisted_xml_fixtures() { - let mut exported_payload = Vec::new(); - archive - .by_name(&format!("{BUCKET}/{config_file}")) - .unwrap_or_else(|_| panic!("cluster export must still contain {config_file}")) - .read_to_end(&mut exported_payload) - .unwrap_or_else(|_| panic!("read exported {config_file}")); - assert_eq!(exported_payload, payload, "one bad bucket must not change another bucket's export"); - } - assert!( - archive.by_name(&format!("{BUCKET}/{EXPORT_UNREADABLE_MANIFEST}")).is_err(), - "a bucket whose configurations all read must carry no unreadable marker" - ); - - // The unreadable configuration is named rather than fabricated: no - // targets entry is exported for it at all. - assert!( - archive - .by_name(&format!("{UNREADABLE_BUCKET}/{BUCKET_TARGETS_FILE}")) - .is_err(), - "an unreadable targets blob must never be exported as a configuration" - ); - let mut marker = Vec::new(); - archive - .by_name(&format!("{UNREADABLE_BUCKET}/{EXPORT_UNREADABLE_MANIFEST}")) - .expect("the export must name the configuration it could not read") - .read_to_end(&mut marker) - .expect("read unreadable marker"); - let marker: serde_json::Value = serde_json::from_slice(&marker).expect("the marker must be JSON"); - assert_eq!(marker["bucket"], UNREADABLE_BUCKET); + .expect_err("one unreadable configuration must fail the ordinary whole-cluster backup"); + assert_eq!(*cluster_export.code(), s3s::S3ErrorCode::InternalError); assert_eq!( - marker["unreadable"].as_array().map(Vec::len), - Some(1), - "only the configuration that could not be read may be marked: {marker}" - ); - assert_eq!(marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE); - drop(archive); - - // The marker cannot be misread as a configuration on the way back in: - // the importer switches on configuration names and ignores everything - // else, so the bucket's stored bytes come through untouched and the - // operator still has to repair them explicitly. - import_archive(cluster_archive).await; - let after_round_trip = metadata_sys::get_config_from_disk(UNREADABLE_BUCKET) - .await - .expect("the marked bucket must still load after the round trip"); - assert_eq!( - after_round_trip.bucket_targets_config_json, MINIO_ARRAY_TARGETS, - "importing the marker must not overwrite or fabricate the bucket's targets configuration" + metadata_sys::get_config_from_disk(UNREADABLE_BUCKET) + .await + .expect("failed export must leave targets untouched") + .bucket_targets_config_json, + MINIO_ARRAY_TARGETS ); } } diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 00a7ca6df..40205b703 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -19,6 +19,7 @@ use crate::admin::runtime_sources::{ AppContext, app_context_from_req, current_notification_system_for_context, current_replication_pool_handle, current_replication_stats_handle_for_context, current_runtime_port, object_store_from_req, }; +use crate::admin::storage_api::AdminVersioningConfigExt as _; use crate::admin::storage_api::bucket::metadata::BUCKET_TARGETS_FILE; use crate::admin::storage_api::bucket::metadata_sys; use crate::admin::storage_api::bucket::metadata_sys::get_replication_config; @@ -29,7 +30,7 @@ use crate::admin::storage_api::bucket::replication::{REMOTE_TARGET_READ_ONLY_HIS use crate::admin::storage_api::bucket::target::{ BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat, duration_from_secs_or_nanos, }; -use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys, UnreadableTargetsPolicy}; +use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys}; use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::storage_api::error::StorageError; @@ -71,6 +72,95 @@ const EVENT_ADMIN_REMOTE_TARGET_STATE: &str = "admin_remote_target_state"; /// repaired first, then the rule is set. const REPLACE_UNREADABLE_TARGETS_PARAM: &str = "replace-unreadable"; +fn parse_remote_target_write_modes(uri: &http::Uri) -> S3Result<(bool, bool)> { + let mut update = None; + let mut replace_unreadable = None; + for (key, value) in url::form_urlencoded::parse(uri.query().unwrap_or_default().as_bytes()) { + let mode = match key.as_ref() { + "update" => &mut update, + REPLACE_UNREADABLE_TARGETS_PARAM => &mut replace_unreadable, + _ => continue, + }; + if mode.is_some() { + return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "duplicate remote target write mode")); + } + *mode = Some(match value.as_ref() { + "true" => true, + "false" => false, + _ => { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "remote target write modes must be true or false", + )); + } + }); + } + let update = update.unwrap_or(false); + let replace_unreadable = replace_unreadable.unwrap_or(false); + if update && replace_unreadable { + return Err(s3_error!(InvalidRequest, "replace-unreadable requires a complete target create request")); + } + Ok((update, replace_unreadable)) +} + +/// Repair decisions use the disk snapshot protected by the metadata transaction, +/// never the stale target cache retained after an unreadable configuration load. +async fn persist_remote_target_repair(bucket: &str, mut target: BucketTarget, incarnation: uuid::Uuid) -> S3Result { + let mut discarded_unreadable = false; + let mut target_error = None; + let updated = metadata_sys::update_config_with(bucket, BUCKET_TARGETS_FILE, |metadata| { + if metadata.bucket_incarnation_id != incarnation { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + if target.target_type == BucketTargetType::ReplicationService + && !metadata.versioning_config.as_ref().is_some_and(|config| config.enabled()) + { + target_error = Some(BucketTargetError::BucketReplicationSourceNotVersioned { + bucket: bucket.to_string(), + }); + return Err(StorageError::other("source bucket versioning changed before target repair")); + } + discarded_unreadable = metadata.bucket_targets_unreadable(); + let mut targets = metadata.bucket_target_config.clone().unwrap_or_default(); + let (arn, exists) = BucketTargetSys::remote_arn_for_targets(&targets.targets, &target, &target.deployment_id); + target.arn = arn; + if target.arn.is_empty() { + target_error = Some(BucketTargetError::BucketRemoteArnInvalid { + bucket: bucket.to_string(), + }); + return Err(StorageError::other("remote target ARN is empty")); + } + if !exists { + BucketTargetSys::upsert_target_entry(&mut targets.targets, &target, false).map_err(|error| { + target_error = Some(error); + StorageError::other("remote target merge failed") + })?; + } + serde_json::to_vec(&targets).map_err(StorageError::other) + }) + .await; + if let Some(error) = target_error { + return Err(map_bucket_target_error(error)); + } + updated.map_err(ApiError::from)?; + + // Persistence also publishes the fresh target set under the transaction + // guard. Publishing another snapshot here could undo a concurrent repair. + if discarded_unreadable { + warn!( + event = EVENT_ADMIN_REMOTE_TARGET_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REPLICATION, + action = "set_remote_target", + result = "unreadable_targets_replaced", + bucket = %bucket, + arn = %target.arn, + "admin remote target state" + ); + } + Ok(target.arn) +} + /// Field groups a `set-remote-target?update=true` request may modify, mirroring /// MinIO's `TargetUpdateType` / `GetTargetUpdateOps` query contract: the update /// overlays only the requested groups onto the stored target, so a client can @@ -554,8 +644,7 @@ impl Operation for SetRemoteTargetHandler { return Err(s3_error!(InvalidRequest, "bucket is required")); }; - let update = queries.get("update").is_some_and(|v| v == "true"); - let replace_unreadable = queries.get(REPLACE_UNREADABLE_TARGETS_PARAM).is_some_and(|v| v == "true"); + let (update, replace_unreadable) = parse_remote_target_write_modes(&req.uri)?; warn!("set remote target, bucket: {}, update: {}", bucket, update); @@ -634,6 +723,24 @@ impl Operation for SetRemoteTargetHandler { let bucket_target_sys = BucketTargetSys::get(); + if replace_unreadable { + // Validate the complete replacement before acquiring the metadata + // transaction; remote I/O must not extend the cluster-wide lock. + let incarnation = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + bucket_target_sys + .validate_target(bucket, &remote_target) + .await + .map_err(map_bucket_target_error)?; + // Match ordinary writers: local targets lock, then lifecycle and + // cluster metadata transaction guards acquired by the repair. + let _targets_guard = lock_bucket_targets_metadata(bucket).await; + let arn = persist_remote_target_repair(bucket, remote_target, incarnation).await?; + let arn_str = serde_json::to_string(&arn).map_err(|_| s3_error!(InternalError, "Failed to serialize target ARN"))?; + return Ok(S3Response::new((StatusCode::OK, Body::from(arn_str)))); + } + if !update { let (arn, exist) = bucket_target_sys .get_remote_arn(bucket, Some(&remote_target), remote_target.deployment_id.as_str()) @@ -723,37 +830,10 @@ impl Operation for SetRemoteTargetHandler { let arn = remote_target.arn.clone(); - let unreadable_policy = if replace_unreadable { - UnreadableTargetsPolicy::Replace - } else { - UnreadableTargetsPolicy::FailClosed - }; - let discarding_unreadable_targets = replace_unreadable - && matches!( - bucket_target_sys.list_bucket_targets(bucket).await, - Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) - ); - let targets = bucket_target_sys - .set_target(bucket, &remote_target, update, unreadable_policy) + .set_target(bucket, &remote_target, update) .await .map_err(map_bucket_target_error)?; - - // Audited only where the discard actually happened: the flag alone is - // not an event, on a readable set it changes nothing, and a refused - // write must not leave a record claiming the set was replaced. - if discarding_unreadable_targets { - warn!( - event = EVENT_ADMIN_REMOTE_TARGET_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_REPLICATION, - action = "set_remote_target", - result = "unreadable_targets_replaced", - bucket = %bucket, - arn = %remote_target.arn, - "admin remote target state" - ); - } let json_targets = serde_json::to_vec(&targets).map_err(|e| { error!("Serialization error: {}", e); S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string()) @@ -2712,3 +2792,484 @@ mod tests { assert_eq!(payload["ScannedVersions"], 7); } } + +#[cfg(test)] +mod target_repair_tests { + use super::*; + use crate::admin::runtime_sources::publish_test_app_context; + use crate::admin::storage_api::bucket::metadata::BUCKET_VERSIONING_CONFIG; + use crate::admin::storage_api::bucket::target::BucketTargets; + use http::Extensions; + use http_body_util::BodyExt as _; + use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tracing::instrument::WithSubscriber as _; + + const ACCESS_KEY: &str = "TARGETREPAIRROOT"; + const SECRET_KEY: &str = "targetRepairRootSecret123"; + const BUCKET: &str = "target-repair"; + const BAD_TARGETS: &[u8] = b"unreadable-targets"; + const TARGET_REPAIR_ENV: [(&str, Option<&str>); 3] = [ + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ]; + + struct RemoteTargetServer { + endpoint: String, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for RemoteTargetServer { + fn drop(&mut self) { + self.task.abort(); + } + } + + impl RemoteTargetServer { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind remote target"); + let endpoint = listener.local_addr().expect("remote target address").to_string(); + let task = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.expect("accept remote target request"); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut chunk).await.expect("read remote target request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let head = String::from_utf8_lossy(&request); + let first_line = head.lines().next().unwrap_or_default(); + let body = if first_line.starts_with("HEAD /") { + "" + } else if first_line.starts_with("GET /") && first_line.contains("versioning") { + "Enabled" + } else { + panic!("unexpected remote target request: {first_line}"); + }; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + socket + .write_all(response.as_bytes()) + .await + .expect("reply to remote target request"); + } + }); + Self { endpoint, task } + } + + fn target(&self) -> BucketTarget { + BucketTarget { + source_bucket: BUCKET.to_string(), + endpoint: self.endpoint.clone(), + target_bucket: "remote".to_string(), + target_type: BucketTargetType::ReplicationService, + region: "us-east-1".to_string(), + credentials: Some(TargetCredentials { + access_key: "remote-access".to_string(), + secret_key: "remote-secret".to_string(), + ..Default::default() + }), + ..Default::default() + } + } + } + + async fn test_env() -> (tempfile::TempDir, rustfs_test_utils::TestECStoreEnv) { + let _ = rustfs_credentials::init_global_action_credentials(Some(ACCESS_KEY.to_string()), Some(SECRET_KEY.to_string())); + let temp = tempfile::tempdir().expect("create repair test root"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp.path()) + .disk_count(1) + .build() + .await; + env.make_bucket(BUCKET, true).await; + rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore)) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)) + .await + .expect("build test IAM"); + publish_test_app_context(Arc::new(AppContext::with_default_interfaces( + Arc::clone(&env.ecstore), + iam, + Arc::new(rustfs_kms::KmsServiceManager::new()), + ))); + metadata_sys::update( + BUCKET, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("persist source versioning"); + (temp, env) + } + + fn request(method: Method, query: &str, body: Vec) -> S3Request { + let operation = if method == Method::GET { + "list-remote-targets" + } else { + "set-remote-target" + }; + S3Request { + input: Body::from(body), + method, + uri: format!("/rustfs/admin/v3/{operation}?bucket={BUCKET}&{query}") + .parse() + .expect("admin URI"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: Some(s3s::auth::Credentials { + access_key: ACCESS_KEY.to_string(), + secret_key: s3s::auth::SecretKey::from(SECRET_KEY.to_string()), + }), + region: None, + service: None, + trailing_headers: None, + } + } + + async fn seed_unreadable(env: &rustfs_test_utils::TestECStoreEnv) { + let mut metadata = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read source metadata"); + metadata.bucket_targets_config_json = BAD_TARGETS.to_vec(); + metadata + .save_with_store(Arc::clone(&env.ecstore)) + .await + .expect("persist unreadable targets"); + crate::storage::storage_api::set_bucket_metadata(BUCKET.to_string(), metadata) + .await + .expect("publish unreadable targets"); + assert!(BucketTargetSys::get().list_bucket_targets(BUCKET).await.is_err()); + } + + async fn repair(target: &BucketTarget, query: &str) -> S3Result { + let body = serde_json::to_vec(&remote_target_admin_json(target).expect("serialize target request")) + .expect("encode target request"); + let response = SetRemoteTargetHandler {} + .call(request(Method::PUT, query, body), Params::new()) + .await?; + assert_eq!(response.output.0, StatusCode::OK); + let body = response.output.1.collect().await.expect("collect target ARN").to_bytes(); + Ok(serde_json::from_slice(&body).expect("plain JSON ARN")) + } + + #[tokio::test] + #[serial_test::serial] + async fn repair_existing_cached_target_persists_readable_targets_and_lists() { + temp_env::async_with_vars(TARGET_REPAIR_ENV, async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let mut target = server.target(); + target.arn = "arn:rustfs:replication:us-east-1:cached:remote".to_string(); + let targets = BucketTargets { + targets: vec![target.clone()], + }; + metadata_sys::update(BUCKET, BUCKET_TARGETS_FILE, serde_json::to_vec(&targets).expect("encode cached target")) + .await + .expect("seed cached target"); + seed_unreadable(&env).await; + assert_eq!( + BucketTargetSys::get().get_remote_arn(BUCKET, Some(&target), "").await, + (target.arn.clone(), true) + ); + assert!( + BucketTargetSys::get() + .get_remote_target_client(BUCKET, &target.arn) + .await + .is_some() + ); + + let arn = repair(&target, "replace-unreadable=true").await.expect("repair must commit"); + assert_ne!(arn, target.arn); + assert!( + BucketTargetSys::get() + .get_remote_target_client(BUCKET, &target.arn) + .await + .is_none() + ); + assert!(BucketTargetSys::get().get_remote_target_client(BUCKET, &arn).await.is_some()); + let persisted = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read repaired metadata"); + assert!(!persisted.bucket_targets_unreadable()); + let targets = persisted.bucket_target_config.expect("decode persisted repair"); + assert_eq!(targets.targets.len(), 1); + assert_eq!(targets.targets[0].arn, arn); + assert_eq!( + targets.targets[0] + .credentials + .as_ref() + .expect("persist credentials") + .secret_key, + "remote-secret" + ); + let list = ListRemoteTargetHandler {} + .call(request(Method::GET, "", Vec::new()), Params::new()) + .await + .expect("list repaired targets"); + assert_eq!(list.output.0, StatusCode::OK); + let listed: serde_json::Value = + serde_json::from_slice(&list.output.1.collect().await.expect("collect target list").to_bytes()) + .expect("decode list"); + assert_eq!(listed.as_array().expect("targets list").len(), 1); + assert_eq!(listed[0]["arn"], arn); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn repair_partial_update_and_invalid_flags_preserve_persisted_bytes() { + temp_env::async_with_vars(TARGET_REPAIR_ENV, async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let mut target = server.target(); + target.arn = "arn:rustfs:replication:us-east-1:cached:remote".to_string(); + metadata_sys::update( + BUCKET, + BUCKET_TARGETS_FILE, + serde_json::to_vec(&BucketTargets { + targets: vec![target.clone()], + }) + .expect("encode cached target"), + ) + .await + .expect("seed cached target"); + seed_unreadable(&env).await; + let file = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read source metadata") + .save_file_path(); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read original bytes"); + for query in [ + "update=true&replace-unreadable=true", + "replace-unreadable=false&replace-unreadable=true", + "replace-unreadable=true&replace-unreadable=false", + "replace-unreadable=true&replace%2dunreadable=true", + "replace-unreadable=TRUE", + "replace-unreadable=1", + "replace-unreadable=", + "update=true&update=false", + "update=invalid", + ] { + assert_eq!( + repair(&target, query).await.expect_err("invalid repair must fail").code(), + &S3ErrorCode::InvalidRequest + ); + let after = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read unchanged bytes"); + assert_eq!(after, before, "rejected opt-in must not rewrite metadata"); + } + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn repair_with_stale_unreadable_cache_preserves_another_committed_repair() { + temp_env::async_with_vars(TARGET_REPAIR_ENV, async { + let (_temp, env) = test_env().await; + let first = RemoteTargetServer::start().await; + let second = RemoteTargetServer::start().await; + seed_unreadable(&env).await; + let first_arn = repair(&first.target(), "replace-unreadable=true") + .await + .expect("commit first repair"); + // Model another node which still retains the original unreadable + // verdict when it begins its repair after this commit. + BucketTargetSys::get().mark_targets_unreadable(BUCKET).await; + let second_arn = repair(&second.target(), "replace-unreadable=true") + .await + .expect("merge second repair"); + let persisted = metadata_sys::get_config_from_disk(BUCKET).await.expect("read both repairs"); + let targets = persisted.bucket_target_config.expect("decode both repairs"); + assert_eq!(targets.targets.len(), 2); + assert!(targets.targets.iter().any(|target| target.arn == first_arn)); + assert!(targets.targets.iter().any(|target| target.arn == second_arn)); + assert_eq!( + BucketTargetSys::get() + .list_bucket_targets(BUCKET) + .await + .expect("published repair") + .targets + .len(), + 2 + ); + assert_eq!( + repair(&first.target(), "replace-unreadable=true") + .await + .expect("idempotent repair"), + first_arn + ); + assert_eq!( + metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read repeated repair") + .bucket_target_config + .expect("decode repeated repair") + .targets + .len(), + 2 + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn repair_transaction_rejects_a_bucket_recreated_after_target_validation() { + temp_env::async_with_vars( + TARGET_REPAIR_ENV, + Box::pin(async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let target = server.target(); + let incarnation = metadata_sys::capture_bucket_metadata_incarnation(BUCKET) + .await + .expect("capture original bucket"); + BucketTargetSys::get() + .validate_target(BUCKET, &target) + .await + .expect("validate original source and remote target"); + + env.ecstore + .delete_bucket(BUCKET, &Default::default()) + .await + .expect("delete original bucket"); + env.make_bucket(BUCKET, true).await; + seed_unreadable(&env).await; + let recreated = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("load recreated bucket"); + assert_ne!(recreated.bucket_incarnation_id, incarnation); + let file = recreated.save_file_path(); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read recreated bucket bytes"); + + let error = persist_remote_target_repair(BUCKET, target, incarnation) + .await + .expect_err("validation of a deleted bucket must not authorize repair of its replacement"); + assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read rejected incarnation repair bytes"), + before + ); + }), + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn repair_transaction_rejects_versioning_suspended_after_target_validation() { + temp_env::async_with_vars( + TARGET_REPAIR_ENV, + Box::pin(async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + let target = server.target(); + seed_unreadable(&env).await; + let incarnation = metadata_sys::capture_bucket_metadata_incarnation(BUCKET) + .await + .expect("capture source bucket"); + BucketTargetSys::get() + .validate_target(BUCKET, &target) + .await + .expect("validate versioned source and remote target"); + + metadata_sys::update( + BUCKET, + BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec(), + ) + .await + .expect("suspend source versioning after validation"); + let suspended = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("load suspended source bucket"); + assert_eq!(suspended.bucket_incarnation_id, incarnation); + assert!(!suspended.versioning_config.as_ref().expect("persisted versioning").enabled()); + let file = suspended.save_file_path(); + let before = crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read suspended bucket bytes"); + + let error = persist_remote_target_repair(BUCKET, target, incarnation) + .await + .expect_err("a target validated before suspension must not be committed"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read rejected versioning repair bytes"), + before + ); + }), + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn failed_repair_transaction_never_reports_a_successful_replacement() { + temp_env::async_with_vars(TARGET_REPAIR_ENV, async { + let (_temp, env) = test_env().await; + let server = RemoteTargetServer::start().await; + seed_unreadable(&env).await; + let target = server.target(); + BucketTargetSys::get() + .validate_target(BUCKET, &target) + .await + .expect("remote validation must succeed before injecting the metadata failure"); + let file = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("read source metadata") + .save_file_path(); + // Keep the source versioning and unreadable-target caches intact, + // but make the transaction's fresh disk load fail. + let corrupt = b"invalid metadata envelope".to_vec(); + env.put_object_bytes(".rustfs.sys", &file, corrupt.clone()).await; + assert!(metadata_sys::get_config_from_disk(BUCKET).await.is_err()); + let log = tempfile::NamedTempFile::new().expect("create captured log"); + let writer = log.reopen().expect("open captured log writer"); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(writer) + .finish(); + let error = repair(&target, "replace-unreadable=true") + .with_subscriber(subscriber) + .await + .expect_err("repair must fail on the unreadable metadata envelope"); + assert_eq!(error.code(), &S3ErrorCode::InternalError); + let lines = std::fs::read_to_string(log.path()).expect("read captured log"); + assert!( + !lines.contains("unreadable_targets_replaced"), + "a failed transaction must not claim success: {lines}" + ); + assert_eq!( + crate::admin::storage_api::config::read_admin_config(Arc::clone(&env.ecstore), &file) + .await + .expect("read failed repair bytes"), + corrupt + ); + }) + .await; + } +} diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 111fe52ce..ecf18cd6e 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -212,7 +212,6 @@ pub(crate) mod bucket_target_sys { pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError; pub(crate) type SsecPassthroughCapability = super::ecstore_bucket::bucket_target_sys::SsecPassthroughCapability; pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient; - pub(crate) type UnreadableTargetsPolicy = super::ecstore_bucket::bucket_target_sys::UnreadableTargetsPolicy; } pub(crate) mod lifecycle { @@ -299,10 +298,7 @@ pub(crate) mod metadata_sys { use std::sync::Arc; use rustfs_policy::policy::BucketPolicy; - use s3s::dto::{ - BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ServerSideEncryptionConfiguration, - Tagging, VersioningConfiguration, - }; + use s3s::dto::{ObjectLockConfiguration, ServerSideEncryptionConfiguration, VersioningConfiguration}; use time::OffsetDateTime; use super::Result; @@ -320,6 +316,13 @@ pub(crate) mod metadata_sys { crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } + pub(crate) async fn update_config_with(bucket: &str, config_file: &str, mutate: F) -> Result + where + F: FnOnce(&BucketMetadata) -> Result> + Send, + { + super::ecstore_bucket::metadata_sys::update_config_with(bucket, config_file, mutate).await + } + pub(crate) async fn update_if_incarnation( bucket: &str, config_file: &str, @@ -412,14 +415,6 @@ pub(crate) mod metadata_sys { serde_json::from_slice(&metadata.bucket_targets_config_json).map_err(super::Error::other) } - pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await - } - - pub(crate) async fn get_notification_config(bucket: &str) -> Result> { - super::ecstore_bucket::metadata_sys::get_notification_config(bucket).await - } - pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await } @@ -442,10 +437,6 @@ pub(crate) mod metadata_sys { super::ecstore_bucket::metadata_sys::get_sse_config(bucket).await } - pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await - } - pub(crate) async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> { super::ecstore_bucket::metadata_sys::get_versioning_config(bucket).await } diff --git a/rustfs/src/on_demand_migration/azure.rs b/rustfs/src/on_demand_migration/azure.rs index 25d4cb0a1..da904c91e 100644 --- a/rustfs/src/on_demand_migration/azure.rs +++ b/rustfs/src/on_demand_migration/azure.rs @@ -44,8 +44,9 @@ use super::storage_api::HTTPRangeSpec; use super::storage_api::remote_s3_client::RemoteS3ClientError; use hmac::{Hmac, Mac, digest::KeyInit}; use http::{HeaderMap, HeaderValue, Method}; +use percent_encoding::percent_decode_str; use quick_xml::Reader; -use quick_xml::events::Event; +use quick_xml::events::{BytesStart, Event}; use sha2::Sha256; use std::collections::{BTreeMap, HashMap}; use url::Url; @@ -379,13 +380,23 @@ fn parse_list_blobs(xml: &str) -> Result { _ => { let end = start.to_end().into_owned(); let text = leaf_text(&mut reader, end.name())?; + let text = if name == "name" { + decode_list_name(&start, text)? + } else { + text + }; apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); } } } Ok(Event::Empty(empty)) => { let name = local_name(empty.name().as_ref()); - apply_list_field(&name, String::new(), &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); + let text = if name == "name" { + decode_list_name(&empty, String::new())? + } else { + String::new() + }; + apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix); } Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() { "blob" => { @@ -426,6 +437,43 @@ fn parse_list_blobs(xml: &str) -> Result { }) } +/// Azure marks XML-inexpressible blob/prefix names with `Encoded="true"`. +/// Only those names are URI-decoded, once; ordinary percent signs and `+` +/// are part of the key, and NextMarker remains an opaque cursor. +fn decode_list_name(start: &BytesStart<'_>, text: String) -> Result { + let mut encoded = false; + for attribute in start.attributes() { + let attribute = attribute.map_err(|_| SourceError::Other("source listing name has invalid attributes".to_string()))?; + if attribute.key.as_ref() == "Encoded" { + let value = quick_xml::escape::unescape(&attribute.value) + .map_err(|_| SourceError::Other("source listing name has an invalid Encoded attribute".to_string()))?; + encoded = match value.as_ref() { + "true" | "1" => true, + "false" | "0" => false, + _ => return Err(SourceError::Other("source listing name has an invalid Encoded attribute".to_string())), + }; + } + } + if !encoded { + return Ok(text); + } + + // percent_decode_str leaves malformed escapes untouched. Refuse them + // rather than return a different key or replace invalid UTF-8 with U+FFFD. + let mut bytes = text.bytes(); + while let Some(byte) = bytes.next() { + if byte == b'%' + && !(bytes.next().is_some_and(|b| b.is_ascii_hexdigit()) && bytes.next().is_some_and(|b| b.is_ascii_hexdigit())) + { + return Err(SourceError::Other("source listing name has invalid percent encoding".to_string())); + } + } + percent_decode_str(&text) + .decode_utf8() + .map(|name| name.into_owned()) + .map_err(|_| SourceError::Other("source listing name is not valid UTF-8".to_string())) +} + fn apply_list_field( name: &str, text: String, @@ -586,6 +634,13 @@ mod tests { const LAST_PAGE: &str = r#" only.txt1"#; + const ENCODED_NAME_PAGE: &str = r#" +%EF%BF%BE/part%252F+%20%26.txt5 +%EF%BF%BE/part%252F+%20%26.txt5 +%EF%BF%BF%2F +literal%FF+/ +opaque%2F+cursor"#; + const TAGS: &str = r#" envprod @@ -618,6 +673,87 @@ mod tests { let listing = parse_list_blobs(LAST_PAGE).expect("page should parse"); assert_eq!(listing.objects.len(), 1); assert!(listing.next_marker.is_none(), "an empty NextMarker is not a cursor"); + let empty = parse_list_blobs("") + .expect("an empty final page is valid"); + assert!(empty.objects.is_empty()); + assert!(empty.prefixes.is_empty()); + assert!(empty.next_marker.is_none()); + } + + #[test] + fn list_blobs_decodes_only_marked_names_once() { + let listing = parse_list_blobs(ENCODED_NAME_PAGE).expect("encoded names should parse"); + assert_eq!(listing.objects[0].key, "\u{fffe}/part%2F+ &.txt"); + assert_eq!(listing.objects[1].key, "%EF%BF%BE/part%252F+%20%26.txt"); + assert_eq!(listing.prefixes, ["\u{ffff}/", "literal%FF+/"]); + assert_eq!(listing.next_marker.as_deref(), Some("opaque%2F+cursor")); + + for (attribute, text, expected) in [ + ("", "a%2Fb+ &.txt", "a%2Fb+ &.txt"), + ("Encoded=\"false\"", "a%2Fb+ &.txt", "a%2Fb+ &.txt"), + ("Encoded=\"0\"", "a%2Fb+ &.txt", "a%2Fb+ &.txt"), + ("Encoded=\"true\"", "a%2Fb+ &.txt", "a/b+ &.txt"), + ("Encoded=\"1\"", "a%2Fb+ &.txt", "a/b+ &.txt"), + ("Encoded=\"true\"", "a%2Fb+ &.txt", "a/b+ &.txt"), + ("", "%", "%"), + ("Encoded=\"false\"", "%", "%"), + ("", "中文/plain%2F+name%", "中文/plain%2F+name%"), + ("Encoded=\"false\"", "中文/plain%2F+name%", "中文/plain%2F+name%"), + ( + "Encoded=\"true\"", + "%EF%BF%BE%EF%BF%BF/%E4%B8%AD%E6%96%87-%25-%2B-%252F+&-%26amp%3B", + "\u{fffe}\u{ffff}/中文-%-+-%2F+&-&", + ), + ] { + for container in ["Blob", "BlobPrefix"] { + let properties = if container == "Blob" { + "0" + } else { + "" + }; + let xml = format!( + "<{container}>{text}{properties}" + ); + let listing = parse_list_blobs(&xml).expect("valid name"); + if container == "Blob" { + assert_eq!(listing.objects.len(), 1); + assert_eq!(listing.objects[0].key, expected, "{container}: {attribute}, {text}"); + assert_eq!(listing.objects[0].size, 0, "a named zero-byte blob remains valid"); + } else { + assert!(listing.objects.is_empty(), "a prefix-only page remains valid"); + assert_eq!(listing.prefixes, [expected], "{container}: {attribute}, {text}"); + } + } + } + } + + #[test] + fn list_blobs_rejects_invalid_encoded_names_without_returning_partial_entries() { + for name in [ + "%", + "%2", + "%GG", + "%FF", + "%E2%82", + "%C0%AF", + "%ED%A0%80", + "a", + "a", + "", + "a", + ] { + for container in ["Blob", "BlobPrefix"] { + let properties = if container == "Blob" { + "1" + } else { + "" + }; + let xml = format!( + "before1<{container}>{name}{properties}after1opaque%2B+marker" + ); + assert!(matches!(parse_list_blobs(&xml), Err(SourceError::Other(_))), "{container}: {name}"); + } + } } #[test] @@ -906,6 +1042,118 @@ mod tests { ); } + #[tokio::test] + async fn listed_encoded_and_literal_names_get_distinct_source_objects() { + let mut head_headers = blob_headers(); + head_headers.push(("Content-Length", "5".to_string())); + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(200, Vec::new(), ENCODED_NAME_PAGE.to_string()), + ScriptedResponse::new(200, head_headers.clone(), String::new()), + ScriptedResponse::new(200, blob_headers(), "first".to_string()), + ScriptedResponse::new(200, head_headers, String::new()), + ScriptedResponse::new(200, blob_headers(), "other".to_string()), + ]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + let page = backend + .list(&SourceListRequest { + max_keys: 4, + ..Default::default() + }) + .await + .expect("list names"); + assert_eq!(page.objects.len(), 2); + assert_eq!(page.objects[0].key, "\u{fffe}/part%2F+ &.txt"); + assert_eq!(page.objects[1].key, "%EF%BF%BE/part%252F+%20%26.txt"); + assert_eq!(page.common_prefixes, ["\u{ffff}/", "literal%FF+/"]); + for (object, body) in page.objects.iter().zip([b"first", b"other"]) { + let head = backend.head(&object.key).await.expect("head listed object"); + assert_eq!(head.size, object.size); + let got = backend.get(&object.key, None).await.expect("get listed object"); + assert_eq!(got.body.collect().await.expect("source body").into_bytes().as_ref(), body); + } + let recorded = recorded.lock().expect("recorder lock"); + assert_eq!(recorded.len(), 5); + for (requests, expected) in recorded[1..].as_chunks::<2>().0.iter().zip([ + "/legacy/%EF%BF%BE/part%252F+%20&.txt", + "/legacy/%25EF%25BF%25BE/part%25252F+%2520%2526.txt", + ]) { + assert_eq!(requests[0].method, "HEAD"); + assert_eq!(requests[1].method, "GET"); + assert_eq!(requests[0].target, expected); + assert_eq!(requests[1].target, expected); + } + } + + #[tokio::test] + async fn listed_encoded_prefix_and_opaque_marker_round_trip_through_query_encoding() { + const PAGE: &str = r#"%EF%BF%BE%EF%BF%BF/%E4%B8%AD%E6%96%87/%252F%2B%25+%26/opaque%2B+marker"#; + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(200, Vec::new(), PAGE.to_string()), + ScriptedResponse::new(200, Vec::new(), LAST_PAGE.to_string()), + ScriptedResponse::new(200, Vec::new(), LAST_PAGE.to_string()), + ]) + .await; + let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32])); + let first = backend + .list(&SourceListRequest { + delimiter: Some("/"), + max_keys: 1, + ..Default::default() + }) + .await + .expect("list encoded prefix"); + assert!(first.objects.is_empty()); + assert_eq!(first.common_prefixes, ["\u{fffe}\u{ffff}/中文/%2F+%+&/"]); + assert_eq!(first.next_continuation_token.as_deref(), Some("opaque%2B+marker")); + assert!(first.is_truncated); + + let second = backend + .list(&SourceListRequest { + delimiter: Some("/"), + continuation_token: first.next_continuation_token.as_deref(), + max_keys: 1, + ..Default::default() + }) + .await + .expect("continue with the original listing conditions"); + assert!(!second.is_truncated); + assert!(second.next_continuation_token.is_none()); + + let nested = backend + .list(&SourceListRequest { + prefix: Some(&first.common_prefixes[0]), + delimiter: Some("/"), + max_keys: 1, + ..Default::default() + }) + .await + .expect("start a separate listing under the returned logical prefix"); + assert!(!nested.is_truncated); + + let recorded = recorded.lock().expect("recorder lock"); + assert_eq!(recorded.len(), 3); + assert_eq!(recorded[0].method, "GET"); + assert!(!recorded[0].target.contains("marker=")); + assert_eq!(recorded[1].method, "GET"); + assert_eq!( + recorded[1].target, + "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%252B%2Bmarker&maxresults=1" + ); + let request_url = endpoint.join(&recorded[1].target).expect("recorded request URL"); + let query: HashMap<_, _> = request_url.query_pairs().into_owned().collect(); + assert_eq!(query.get("marker").map(String::as_str), Some("opaque%2B+marker")); + assert_eq!(recorded[2].method, "GET"); + assert_eq!( + recorded[2].target, + "/legacy?restype=container&comp=list&prefix=%EF%BF%BE%EF%BF%BF%2F%E4%B8%AD%E6%96%87%2F%252F%2B%25%2B%26%2F&delimiter=%2F&maxresults=1" + ); + let request_url = endpoint.join(&recorded[2].target).expect("recorded prefix request URL"); + let query: HashMap<_, _> = request_url.query_pairs().into_owned().collect(); + assert_eq!(query.get("prefix"), Some(&first.common_prefixes[0])); + assert!(!query.contains_key("marker")); + } + #[tokio::test] async fn tagging_and_probe_address_the_right_resources() { let (endpoint, recorded) = scripted_server(vec![ diff --git a/rustfs/src/on_demand_migration/gcs.rs b/rustfs/src/on_demand_migration/gcs.rs index bce647b97..5b43ca6ec 100644 --- a/rustfs/src/on_demand_migration/gcs.rs +++ b/rustfs/src/on_demand_migration/gcs.rs @@ -561,4 +561,41 @@ mod tests { } } } + + /// GCS states its error code in the response body, which this backend never + /// reads, so every class must follow from the status alone. The classes are + /// what the runtime acts on: only `NotFound` is negative-cached, and only a + /// retryable class may be re-sent rather than counted against the breaker. + /// The 404 row scripts the readable-bucket probe as well, because a GCS + /// object 404 is only a key miss once the bucket has answered + /// (`object_404_requires_a_readable_source_bucket` pins that rule). + #[tokio::test] + async fn gcs_statuses_map_onto_the_shared_error_classes() { + for method in [Method::HEAD, Method::GET] { + for (status, expected, retryable) in [ + (404_u16, "not_found", false), + (403, "access_denied", false), + (401, "access_denied", false), + (429, "throttled", true), + (503, "throttled", true), + (500, "server_error", true), + (502, "server_error", true), + ] { + let mut script = vec![ScriptedResponse::new(status, Vec::new(), String::new())]; + if status == 404 { + script.push(ScriptedResponse::new(200, Vec::new(), "{}".to_string())); + } + let (endpoint, _) = scripted_server(script).await; + let backend = backend(&endpoint); + let result = if method == Method::HEAD { + backend.head("a.txt").await.map(|_| ()) + } else { + backend.get("a.txt", None).await.map(|_| ()) + }; + let err = result.expect_err("a non-2xx status must fail"); + assert_eq!(err.class_label(), expected, "{method} HTTP {status}: {err:?}"); + assert_eq!(err.is_retryable(), retryable, "{method} HTTP {status}: {err:?}"); + } + } + } } diff --git a/scripts/README.md b/scripts/README.md index d76e7ba33..25c45c863 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -46,6 +46,8 @@ their issue closes. | Entry | Status | Purpose | Wiring / docs | |---|---|---|---| +| `diagnose_scanner_enumeration_restart.py` | dev-tool | Strict fixed raw-entry-budget scanner-worker restart diagnostic | [Checkpoint fixture](../docs/testing/scanner-checkpoint-fixture.md) | +| `test_diagnose_scanner_enumeration_restart.py` | dev-tool | Driver report validation and positive convergence oracle tests | Python unittest; same guide | | `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` | | `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` | | `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` | diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index ef3f15107..3415167a9 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -12,11 +12,15 @@ import sys import tempfile import tomllib import unittest +import uuid +import xml.etree.ElementTree as ET from datetime import datetime, timezone from unittest import mock from pathlib import Path from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json + ROOT = Path(__file__).resolve().parents[1] SCHEDULED_ALERT_WORKFLOWS = tuple( @@ -873,6 +877,187 @@ def check_core_listing(root: Path, listing: Path) -> list[str]: return [f"cannot read core nextest listing: {error}"] +def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> int: + require(type(value) is int and minimum <= value <= maximum, f"invalid integer {name}") + return value + + +def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None: + """Record an existing build; this command never builds or runs a test.""" + require(not directory.exists(), "scanner/heal run directory must be new") + require(not subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, text=True).strip(), + "commit tracked source changes before creating evidence") + builds = {} + for label, path in (("binary", binary), ("test_binary", test_binary)): + path = path.resolve(strict=True) + require(path.is_file() and os.access(path, os.X_OK), f"missing executable {label}") + builds[label] = {"path": str(path), "sha256": digest(path)} + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + require(re.fullmatch(r"[0-9a-f]{40}", revision), "invalid source revision") + version = subprocess.check_output([builds["binary"]["path"], "--version"], text=True, timeout=30) + embedded_revision = re.search(r"^git commit\s*:\s*([0-9a-f]{40})\s*$", version, re.MULTILINE) + embedded_status = re.search(r"^git status\s*:\s*(.*)\Z", version, re.MULTILINE | re.DOTALL) + require(embedded_revision is not None and embedded_revision[1] == revision, "server binary source revision mismatch") + require(embedded_status is not None and not embedded_status[1].strip(), "server binary was built from dirty/unknown source") + lock_blob = subprocess.check_output(["git", "hash-object", "Cargo.lock"], cwd=root, text=True).strip() + require(re.fullmatch(r"[0-9a-f]{40}", lock_blob), "invalid Cargo.lock identity") + features = os.environ.get("RUSTFS_E2E_EXPECTED_FEATURES") + require(features is not None, "set RUSTFS_E2E_EXPECTED_FEATURES to the compiled e2e crate feature set") + features = ",".join(sorted(set(filter(None, (feature.strip() for feature in features.split(",")))))) + require(all(re.fullmatch(r"[a-z0-9-]+", feature) for feature in features.split(",") if feature), "invalid expected features") + directory.mkdir(parents=True) + write_json(directory / "run.json", {"schema": 1, "run_id": uuid.uuid4().hex, + "source_revision": revision, + "binary_source_revision": embedded_revision[1], + "test_build": {"source_revision": revision, "dirty": False, + "lock_blob": lock_blob, "features": features}, + "started_at": datetime.now(timezone.utc).timestamp(), **builds}) + + +def finish_scanner_heal_receipt(directory: Path, exit_code: int) -> None: + require(type(exit_code) is int and 0 <= exit_code <= 255, "invalid test exit code") + require(not (directory / "execution.json").exists(), "execution receipt already exists") + run = read_json(directory / "run.json") + artifacts = {} + for name in ("listing.json", "junit.xml", "background-target-restart.json"): + path = directory / name + if exit_code != 0 and not path.exists(): + continue + require(path.is_file() and 0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}") + require(path.stat().st_mtime >= run["started_at"], f"stale {name}") + artifacts[name] = digest(path) + write_json(directory / "execution.json", {"run_id": run["run_id"], "exit_code": exit_code, + "finished_at": datetime.now(timezone.utc).timestamp(), + "artifacts": artifacts}) + + +def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> list[str]: + """Validate one actual case, or fail the release while required lanes are pending.""" + try: + registry = read_json(root / ".config/scanner-heal-required-tests.json") + evidence_integer(registry.get("schema"), "registry schema", 1, 1) + require(registry.get("cases"), "invalid scanner/heal registry") + selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]} + run = read_json(directory / "run.json") + execution = read_json(directory / "execution.json") + evidence_integer(run.get("schema"), "run schema", 1, 1) + require(re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity") + require(re.fullmatch(r"[0-9a-f]{40}", run["source_revision"]), "invalid source revision") + require(run.get("binary_source_revision") == run["source_revision"], "server source provenance missing") + expected_build = run["test_build"] + require(expected_build["source_revision"] == run["source_revision"] and expected_build["dirty"] is False, + "test source provenance missing") + require(re.fullmatch(r"[0-9a-f]{40}", expected_build["lock_blob"]), "invalid test lockfile identity") + require(isinstance(expected_build["features"], str), "missing test features") + require(execution.get("run_id") == run["run_id"], "execution belongs to another run") + require(type(execution.get("exit_code")) is int and execution["exit_code"] == 0, "test command failed or did not run") + number(run["started_at"], "started_at", 1) + number(execution["finished_at"], "finished_at", run["started_at"]) + for label in ("binary", "test_binary"): + require(sha(run[label]["sha256"]) and digest(Path(run[label]["path"])) == run[label]["sha256"], + f"{label} changed or missing") + for name in ("listing.json", "junit.xml"): + path = directory / name + require(0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}") + require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], f"{name} outside run window") + require(digest(path) == execution["artifacts"][name], f"{name} hash mismatch") + suites = read_json(directory / "listing.json")["rust-suites"] + xml = (directory / "junit.xml").read_bytes() + require(b" 0, "missing expected bytes") + require(type(obj["actual_bytes"]) is int and obj["actual_bytes"] == obj["expected_bytes"], "S3 body length mismatch") + require(sha(obj["expected_sha256"]) and obj["actual_sha256"] == obj["expected_sha256"], "S3 body digest mismatch") + physical = obj["physical"] + if obj["expected_physical"] is not None: + require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest") + for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []): + data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16) + parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16) + require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], + "EC geometry differs from this case's single set") + evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity) + require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata") + parts = physical["expected_part_numbers"] + require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage") + require(all(type(part) is int and part > 0 for part in parts) and len(set(parts)) == len(parts), + "invalid physical part identity") + require({str(part) for part in parts} == set(physical["present_part_fingerprints"]), "target shard parts missing") + for part in physical["present_part_fingerprints"].values(): + require(type(part["size"]) is int and part["size"] > 0 and sha(part["sha256"]), "invalid target shard fingerprint") + node_listings = oracle["node_listings"] + require(isinstance(node_listings, list) and len(node_listings) == requirement["topology"]["nodes"], + "missing per-node S3 listing") + require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings), + "S3 listing differs from object oracle") + if case_id == "release": + errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items()) + return errors + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + return [f"scanner/heal evidence rejected: {error}"] + + def validate(root: Path) -> list[str]: errors: list[str] = [] errors.extend(check_core_fixtures(root)) @@ -1022,6 +1207,213 @@ class SelfTests(unittest.TestCase): with mock.patch(__name__ + ".check_quick_checks", return_value=[error]): self.assertIn(error, validate(ROOT)) + def scanner_heal_fixture(self, directory: Path) -> tuple[Path, Path]: + """Parser fixtures only; these files are never runtime evidence.""" + root, run_dir = directory / "repo", directory / "run" + (root / ".config").mkdir(parents=True) + run_dir.mkdir() + registry = read_json(ROOT / ".config/scanner-heal-required-tests.json") + write_json(root / ".config/scanner-heal-required-tests.json", registry) + requirement = registry["cases"]["background-target-restart"] + binary = directory / "fake-binary" + binary.write_bytes(b"parser fixture, not a real build") + binary.chmod(0o700) + build = {"path": str(binary), "sha256": digest(binary)} + write_json(run_dir / "run.json", {"schema": 1, "run_id": "a" * 32, "source_revision": "b" * 40, + "binary_source_revision": "b" * 40, + "test_build": {"source_revision": "b" * 40, "dirty": False, + "lock_blob": "c" * 40, "features": "default"}, + "started_at": datetime.now(timezone.utc).timestamp() - 1, + "binary": build, "test_binary": build}) + write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: { + "binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target", + "testcases": { + requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}} + }}}}) + (run_dir / "junit.xml").write_text( + f'') + physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", + "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1], + "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, + "inline_data_fingerprint": None} + obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, + "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, + "expected_physical": physical, "physical": physical} + objects = [dict(obj, key=f"object-{index}") for index in range(9)] + objects[-1] = dict(objects[-1], expected_physical=None) + write_json(run_dir / "background-target-restart.json", { + "schema": 1, "evidence": "process-restart", "case": "background-target-restart", + "run_id": "a" * 32, "source_revision": "b" * 40, + "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40, + "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""}, + "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"], + "topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11, + "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4, + }) + finish_scanner_heal_receipt(run_dir, 0) + return root, run_dir + + def test_scanner_heal_case_does_not_approve_pending_release(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), []) + errors = check_scanner_heal_evidence(root, run_dir, "release") + self.assertEqual(len(errors), 21) + self.assertTrue(any(error.startswith("pending R-E:") for error in errors)) + self.assertTrue(any(error.startswith("pending R-D:") for error in errors)) + self.assertTrue(any(error.startswith("pending R-L:") for error in errors)) + + def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None: + for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale", + "hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + if fault == "exit": + receipt = read_json(run_dir / "execution.json") + receipt["exit_code"] = 42 + write_json(run_dir / "execution.json", receipt) + elif fault == "missing": + path.unlink() + elif fault == "zero": + (run_dir / "junit.xml").write_text("") + elif fault in ("skipped", "failed", "retry"): + junit = run_dir / "junit.xml" + tag = {"skipped": "skipped", "failed": "failure", "retry": "rerunFailure"}[fault] + junit.write_text(junit.read_text().replace("/>", f"><{tag}/>")) + elif fault in ("filtered", "ignored"): + listing = read_json(run_dir / "listing.json") + case = next(iter(listing["rust-suites"]["e2e_test"]["testcases"].values())) + case["ignored"] = fault == "ignored" + case["filter-match"]["status"] = "mismatch" if fault == "filtered" else "matches" + write_json(run_dir / "listing.json", listing) + elif fault == "stale": + os.utime(path, (1, 1)) + elif fault == "hash": + path.write_text(path.read_text() + " ") + elif fault == "binary": + Path(read_json(run_dir / "run.json")["binary"]["path"]).write_bytes(b"another build") + else: + if fault == "synthetic": + oracle["evidence"] = "synthetic" + elif fault == "wrong-run": + oracle["run_id"] = "f" * 32 + elif fault == "same-pid": + oracle["pid_after"] = oracle["pid_before"] + elif fault == "body": + oracle["objects"][0]["actual_sha256"] = "e" * 64 + elif fault == "parts": + oracle["objects"][0]["physical"]["present_part_fingerprints"] = {} + elif fault == "listing": + oracle["node_listings"][0] = [] + elif fault == "topology": + oracle["topology"] = {"nodes": 3, "drives_per_node": 4} + write_json(path, oracle) + if fault not in ("exit", "missing", "stale", "hash", "binary"): + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault) + + def test_scanner_heal_receipts_reject_reuse_and_missing_builds(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + with self.assertRaisesRegex(ValueError, "already exists"): + finish_scanner_heal_receipt(run_dir, 0) + with self.assertRaisesRegex(ValueError, "must be new"): + begin_scanner_heal_receipt(root, run_dir, Path("missing"), Path("missing")) + with mock.patch("subprocess.check_output", side_effect=["", "b" * 40]): + with self.assertRaises(FileNotFoundError): + begin_scanner_heal_receipt(root, Path(tmp) / "new-run", Path(tmp) / "missing", Path(tmp) / "missing") + + def test_scanner_heal_begin_requires_embedded_source_provenance(self) -> None: + for kind in ("current", "stale", "dirty", "unknown"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + root, _ = self.scanner_heal_fixture(Path(tmp)) + sources = root / "crates/e2e_test/src" + sources.mkdir(parents=True) + (sources / "heal_erasure_disk_rebuild_test.rs").write_bytes(b"oracle source") + (sources / "chaos.rs").write_bytes(b"census source") + revision = "c" * 40 if kind == "stale" else "b" * 40 + version = f"rustfs\ngit commit : {revision}\ngit status :\n" + if kind == "dirty": + version += "modified source\n" + if kind == "unknown": + version = "rustfs without build provenance" + with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version, "c" * 40]), \ + mock.patch.dict(os.environ, {"RUSTFS_E2E_EXPECTED_FEATURES": "default"}): + directory = Path(tmp) / "fresh" + binary = Path(tmp) / "fake-binary" + if kind == "current": + begin_scanner_heal_receipt(root, directory, binary, binary) + self.assertEqual(read_json(directory / "run.json")["binary_source_revision"], "b" * 40) + else: + with self.assertRaisesRegex(ValueError, "server binary"): + begin_scanner_heal_receipt(root, directory, binary, binary) + self.assertFalse(directory.exists()) + + def test_scanner_heal_rejects_copied_junit_and_wrong_suite_build(self) -> None: + for fault in ("old-junit", "missing-time", "wrong-binary", "common-source", "lockfile", "features", "dirty-build"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + if fault in ("old-junit", "missing-time"): + path = run_dir / "junit.xml" + xml = ET.fromstring(path.read_bytes()) + testcase = next(xml.iter("testcase")) + if fault == "old-junit": + testcase.set("timestamp", "2000-01-01T00:00:00.000Z") + else: + del testcase.attrib["timestamp"] + # Rewriting/copying gives an old execution a fresh mtime. + path.write_bytes(ET.tostring(xml)) + elif fault == "wrong-binary": + path = run_dir / "listing.json" + listing = read_json(path) + another = Path(tmp) / "another-binary" + another.write_bytes(Path(tmp, "fake-binary").read_bytes()) + listing["rust-suites"]["e2e_test"]["binary-path"] = str(another) + write_json(path, listing) + else: + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + key, value = {"common-source": ("source_revision", "f" * 40), "lockfile": ("lock_blob", "f" * 40), + "features": ("features", "default,sftp"), "dirty-build": ("dirty", True)}[fault] + oracle["test_build"][key] = value + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault) + + def test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers(self) -> None: + valid = {"schema": 1, "nodes": 4, "drives_per_node": 1, "pid_before": 10, "pid_after": 11, + "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2} + cases = [(field, value) for field, correct in valid.items() for value in (True, float(correct))] + cases += [("erasure_index", 5), ("erasure_index", 0), ("pid_after", -1)] + for field, value in cases: + with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + if field in ("nodes", "drives_per_node"): + oracle["topology"][field] = value + elif field in ("erasure_index", "data_blocks", "parity_blocks"): + oracle["objects"][-1]["physical"][field] = value + else: + oracle[field] = value + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart")) + for filename in ("run.json", ".config/scanner-heal-required-tests.json"): + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = (root if filename.startswith(".config") else run_dir) / filename + content = read_json(path) + content["schema"] = True + write_json(path, content) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart")) + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -1664,6 +2056,25 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"]): + try: + if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal": + begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4])) + return 0 + if len(sys.argv) == 4 and sys.argv[1] == "--finish-scanner-heal": + finish_scanner_heal_receipt(Path(sys.argv[2]), int(sys.argv[3])) + return 0 + if len(sys.argv) == 4 and sys.argv[1] == "--check-scanner-heal": + errors = check_scanner_heal_evidence(ROOT, Path(sys.argv[2]), sys.argv[3]) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + if not errors: + print(f"Case evidence verified: {sys.argv[3]}; this does not approve release") + return 1 if errors else 0 + raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, or --check-scanner-heal DIR CASE|release") + except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if len(sys.argv) == 3 and sys.argv[1] == "--check-core": errors = check_core_listing(ROOT, Path(sys.argv[2])) for error in errors: diff --git a/scripts/diagnose_scanner_enumeration_restart.py b/scripts/diagnose_scanner_enumeration_restart.py new file mode 100644 index 000000000..951c164a7 --- /dev/null +++ b/scripts/diagnose_scanner_enumeration_restart.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Strict restart diagnostic using the real scanner libtest worker, not a walker model.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys + +WORKER = "scanner_folder::tests::enumeration_restart::enumeration_restart_worker" +MAX_REPORT_BYTES = 16384 + + +def bounded_int(low, high): + def parse(value): + number = int(value) + if not low <= number <= high: + raise argparse.ArgumentTypeError(f"must be between {low} and {high}") + return number + return parse + + +def validate_report(report, *, round_number, pid, objects, budget): + if not isinstance(report, dict): + raise ValueError("worker report must be an object") + expected = {"schema": 1, "round": round_number, "pid": pid, + "objects_expected": objects, "raw_entry_budget": budget} + for key, value in expected.items(): + if type(report.get(key)) is not int or report[key] != value: + raise ValueError(f"worker report mismatch: {key}") + for key in ("raw_entries", "raw_name_bytes", "objects_before", "objects_retained", + "versions_retained", "bytes_retained", "objects_processed"): + if type(report.get(key)) is not int or not 0 <= report[key] <= 1048576: + raise ValueError(f"invalid bounded counter: {key}") + if report["raw_entries"] == 0: + raise ValueError("nonempty fixture must observe raw entries; budget hook may not have run") + if report["raw_entries"] > budget: + raise ValueError("raw-entry budget exceeded; no unbudgeted tail is permitted") + if type(report.get("snapshot_complete")) is not bool: + raise ValueError("missing explicit completeness") + if report.get("outcome") not in ("complete", "partial", "cancelled_without_cache"): + raise ValueError("unexpected scanner outcome") + + +def converged(report, objects): + return (report["snapshot_complete"] and report["outcome"] == "complete" + and all(report[key] == objects for key in + ("objects_retained", "versions_retained", "bytes_retained"))) + + +def run(args): + binary = args.test_binary.resolve(strict=True) + listed = subprocess.run([str(binary), WORKER, "--exact", "--list"], + check=True, capture_output=True, text=True, timeout=30) + if f"{WORKER}: test" not in listed.stdout.splitlines(): + raise ValueError("binary does not contain the exact scanner worker test") + workspace = args.output.resolve() + workspace.mkdir() # Refuse reuse/overwrite of previous evidence or customer data. + reports = [] + for round_number in range(args.rounds): + request = {"workspace": str(workspace), "objects": args.objects, + "raw_entry_budget": args.raw_entry_budget, "round": round_number} + request_path = workspace / "request.json" + request_path.write_text(json.dumps(request), encoding="utf-8") + env = dict(os.environ, RUSTFS_ENUMERATION_REQUEST=str(request_path), + RUST_MIN_STACK="4194304", NO_PROXY="localhost,127.0.0.1,::1", + no_proxy="localhost,127.0.0.1,::1") + with subprocess.Popen([str(binary), WORKER, "--exact", "--test-threads=1"], + env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) as worker: + try: + status = worker.wait(timeout=args.timeout) + except subprocess.TimeoutExpired: + worker.kill() + worker.wait() + raise ValueError(f"worker round {round_number} timed out") from None + if status: + raise ValueError(f"real scanner worker round {round_number} exited {status}") + report_path = workspace / f"round-{round_number}.json" + with report_path.open("rb") as handle: + raw = handle.read(MAX_REPORT_BYTES + 1) + if len(raw) > MAX_REPORT_BYTES: + raise ValueError("oversized worker report") + report = json.loads(raw) + validate_report(report, round_number=round_number, pid=worker.pid, + objects=args.objects, budget=args.raw_entry_budget) + if reports and report["objects_before"] != reports[-1]["objects_retained"]: + raise ValueError("cache coverage did not survive the process boundary") + reports.append(report) + print(json.dumps(report, sort_keys=True), flush=True) + if converged(report, args.objects): + print("PASS: bounded scanner-worker restart convergence for this fixture only") + return 0 + print("FAIL: fixed-budget restart convergence not established; R-E gate remains unmet", + file=sys.stderr) + return 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--test-binary", type=Path, required=True, + help="compiled rustfs-scanner libtest executable") + parser.add_argument("--output", type=Path, required=True, help="new evidence directory (must not exist)") + parser.add_argument("--objects", type=bounded_int(1, 1024), default=128) + parser.add_argument("--raw-entry-budget", type=bounded_int(1, 4096), default=8) + parser.add_argument("--rounds", type=bounded_int(1, 64), default=8) + parser.add_argument("--timeout", type=bounded_int(1, 120), default=60, + help="per-worker watchdog seconds, not the scan work budget") + args = parser.parse_args() + try: + return run(args) + except (OSError, ValueError, subprocess.SubprocessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index 70fb05c21..e728dc965 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -8,6 +8,7 @@ import json import math import os from pathlib import Path +import select import shutil import signal import subprocess @@ -126,28 +127,110 @@ def validate_manifest(manifest): require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs") +class OwnedCommand: + """Keep the session leader unreaped until its group's last signal is sent.""" + + def __init__(self, args, log): + require(sys.platform == "darwin" or hasattr(os, "waitid"), "non-reaping child observation is unavailable") + self.args, self.status = args, None + self.queue = select.kqueue() if sys.platform == "darwin" else None + self.process = None + read_gate, write_gate = os.pipe() + try: + # The shell has already exec'd when Popen returns. Gate the target + # until kqueue is registered; preexec_fn would deadlock Popen here. + gate = f'read -r _scanner_gate <&{read_gate} || exit 125; exec {read_gate}<&-; exec "$@"' + self.process = subprocess.Popen(["bash", "-c", gate, "scanner-abba", *args], pass_fds=(read_gate,), + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + if self.queue is not None: + # Darwin NOTE_EXITSTATUS is not exposed by Python's select constants. + event = select.kevent(self.process.pid, filter=select.KQ_FILTER_PROC, + flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, + fflags=select.KQ_NOTE_EXIT | 0x04000000) + self.queue.control([event], 0, 0) + os.write(write_gate, b"\n") + except BaseException: + try: + if self.process is not None: + try: + self._signal_group(signal.SIGKILL) + finally: + self.process.wait(timeout=10) + finally: + if self.queue is not None: + self.queue.close() + raise + finally: + os.close(read_gate) + os.close(write_gate) + + def wait(self, timeout): + if self.status is not None: + return self.status + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(self.args, timeout) + if self.queue is not None: + events = self.queue.control(None, 1, remaining) + if events: + self.status = os.waitstatus_to_exitcode(events[0].data) + return self.status + else: + result = os.waitid(os.P_PID, self.process.pid, os.WEXITED | os.WNOWAIT | os.WNOHANG) + if result is not None: + self.status = result.si_status if result.si_code == os.CLD_EXITED else -result.si_status + return self.status + time.sleep(min(0.05, remaining)) + + def _signal_group(self, sig): + try: + os.killpg(self.process.pid, sig) + return True + except ProcessLookupError: + return False + + def finish(self, terminate=False): + if self.process.returncode is not None: + return self.process.returncode + try: + if terminate: + try: + self._signal_group(signal.SIGTERM) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and self._signal_group(0): + time.sleep(0.05) + finally: + # Keep the PID reserved through the last group signal, even + # when the cleanup grace period itself is interrupted. + self._signal_group(signal.SIGKILL) + finally: + try: + returncode = self.process.wait(timeout=10) + finally: + if self.queue is not None: + self.queue.close() + return returncode + + def invoke(adapter, action, request, timeout): """The adapter writes bounded JSON separately; stderr/stdout remain raw evidence.""" output = request.parent / f"{action}.json" with (request.parent / f"{action}.log").open("wb") as log: - process = subprocess.Popen([str(adapter), action, str(request), str(output)], - stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + process = OwnedCommand([str(adapter), action, str(request), str(output)], log) try: - returncode = process.wait(timeout=timeout) + returncode = process.wait(timeout) if returncode: raise subprocess.CalledProcessError(returncode, [str(adapter), action]) - finally: - if process.poll() != 0: - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait() - return read_json(output) + result = read_json(output) + except BaseException: + process.finish(terminate=True) + raise + else: + # Successful prepare may intentionally leave adapter-owned services. + process.finish() + return result def validate_result(result, request, expected): @@ -188,7 +271,7 @@ def convergence(result): require(window["full_walk_objects"] > 0, "zero full walk reference") require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"], "invalid convergence budget window") - return window["walk_objects"] / window["full_walk_objects"] + return ratio(window["walk_objects"], window["full_walk_objects"], "convergence work") def evaluate(cells): @@ -229,6 +312,7 @@ def evaluate(cells): candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")] p2_pending = any(value is None for value in candidate_p2) passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None) + p2_report = [None if value is None else float(value) for value in p2] inconclusive |= noise or p2_pending if not noise and not passed: failed = True @@ -238,7 +322,7 @@ def evaluate(cells): "p99_regression": float(p99), "throughput_change": float(throughput), "thresholds": {key: float(value) for key, value in thresholds.items()}, "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT), - "p2_post_stop_work_multiples": p2}) + "p2_post_stop_work_multiples": p2_report}) return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons @@ -254,12 +338,12 @@ def collect_live(prepared, request, request_path, adapter): "--samples", str(request["duration_seconds"] // 60 + 1), "--interval-secs", "60", "--out-dir", str(output)] with (request_path.parent / "collector.log").open("wb") as log: - process = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + process = OwnedCommand(args, log) try: started = time.monotonic() result = invoke(adapter, "measure", request_path, request["duration_seconds"] + 300) require(time.monotonic() - started >= request["duration_seconds"], "measurement ended before required window") - require(process.wait(timeout=120) == 0, "scanner collector failed") + require(process.wait(120) == 0, "scanner collector failed") require(output.joinpath("scanner-summary.csv").stat().st_size > 0, "missing collector samples") samples = list((output / "status").glob("scanner-status.*.json")) require(len(samples) == request["duration_seconds"] // 60 + 1, "missing scanner samples") @@ -286,16 +370,7 @@ def collect_live(prepared, request, request_path, adapter): "missing per-host scanner metrics") return result finally: - # Stop telemetry children as well when measurement fails or times out. - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait() + process.finish(terminate=True) def run(manifest, adapter, output, data_root): diff --git a/scripts/test_diagnose_scanner_enumeration_restart.py b/scripts/test_diagnose_scanner_enumeration_restart.py new file mode 100644 index 000000000..efe5f2ddd --- /dev/null +++ b/scripts/test_diagnose_scanner_enumeration_restart.py @@ -0,0 +1,73 @@ +"""Driver contract tests; these do not replace the real scanner diagnostic.""" + +import unittest + +from diagnose_scanner_enumeration_restart import converged, validate_report + + +class ReportTests(unittest.TestCase): + def report(self): + return dict(schema=1, round=0, pid=123, objects_expected=4, raw_entry_budget=16, + raw_entries=8, raw_name_bytes=64, objects_before=0, objects_retained=4, + versions_retained=4, bytes_retained=4, objects_processed=4, + snapshot_complete=True, outcome="complete") + + def validate(self, report): + validate_report(report, round_number=0, pid=123, objects=4, budget=16) + + def test_complete_exact_coverage_satisfies_oracle(self): + report = self.report() + self.validate(report) + self.assertTrue(converged(report, 4)) + + def test_incomplete_or_inexact_coverage_cannot_pass(self): + for key, value in (("snapshot_complete", False), ("objects_retained", 3), + ("versions_retained", 3), ("bytes_retained", 3), ("outcome", "partial")): + with self.subTest(key=key): + report = self.report() + report[key] = value + self.assertFalse(converged(report, 4)) + + def test_wrong_process_or_round_rejected(self): + for key in ("pid", "round", "schema", "raw_entry_budget", "objects_expected"): + with self.subTest(key=key): + report = self.report() + report[key] += 1 + with self.assertRaises(ValueError): + self.validate(report) + + def test_unbudgeted_tail_rejected(self): + report = self.report() + report["raw_entries"] = 17 + with self.assertRaises(ValueError): + self.validate(report) + + def test_complete_coverage_without_entry_observation_rejected(self): + report = self.report() + report["raw_entries"] = 0 + with self.assertRaises(ValueError): + self.validate(report) + + def test_missing_wrong_type_and_negative_counter_rejected(self): + for value in (None, True, -1, "8", 1048577): + with self.subTest(value=value): + report = self.report() + report["raw_entries"] = value + with self.assertRaises(ValueError): + self.validate(report) + + def test_missing_completeness_or_unknown_outcome_rejected(self): + for key in ("snapshot_complete", "outcome"): + report = self.report() + del report[key] + with self.assertRaises(ValueError): + self.validate(report) + + def test_non_object_report_rejected(self): + for report in (None, [], "report"): + with self.assertRaises(ValueError): + self.validate(report) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index 79935f924..5408eb44c 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -3,13 +3,17 @@ import contextlib import copy +import fcntl import io import json import os from pathlib import Path +import shlex +import signal import subprocess import sys import tempfile +import time import unittest from unittest.mock import Mock, patch @@ -20,9 +24,18 @@ def fake_adapter(): action, request_path, output_path = sys.argv[1:] request = harness.read_json(Path(request_path)) fault = os.environ.get("SCANNER_ABBA_TEST_FAULT", "") + if fault == "stubborn-child" and action in ("prepare", "measure"): + marker = Path(request_path).parent / "stubborn.pid" + if os.fork() == 0: + os.execv(sys.executable, [sys.executable, str(Path(__file__).resolve()), "--stubborn-worker", str(marker)]) + wait_for_marker(marker) + if action == "measure": + time.sleep(60) if action == "prepare": result = {"ready": True} elif action == "stop": + if fault == "stubborn-child": + reap_fixture(Path(request_path).parent / "stubborn.pid") result = {"stopped": True} elif action == "oracle": if fault == "oracle-exit": @@ -87,6 +100,36 @@ def fake_adapter(): return 0 +def wait_for_marker(marker): + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if marker.exists() and marker.stat().st_size: + return + time.sleep(0.01) + raise AssertionError("fixture child did not become ready") + + +def child_released(marker, timeout=1): + deadline = time.monotonic() + timeout + with marker.open("r+") as stream: + while True: + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except BlockingIOError: + if time.monotonic() >= deadline: + return False + time.sleep(0.01) + + +def reap_fixture(marker): + if marker.exists() and marker.stat().st_size and not child_released(marker, timeout=0): + # The unique file lock proves the original fixture process still owns this PID. + os.kill(int(marker.read_text()), signal.SIGKILL) + if not child_released(marker, timeout=5): + raise AssertionError("fixture child did not release its process-owned lock") + + class ScannerAbbaTest(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -110,6 +153,112 @@ class ScannerAbbaTest(unittest.TestCase): with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()): return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data") + def test_adapter_timeout_reaps_group_after_parent_exits_on_term(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + with self.assertRaises(subprocess.TimeoutExpired): + harness.invoke(self.adapter, "measure", request, 3) + wait_for_marker(marker) + self.assertTrue(child_released(marker), "TERM-exited parent left its TERM-ignoring child alive") + finally: + reap_fixture(marker) + + def test_collector_failure_reaps_group_after_parent_exits_on_term(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + collector = self.root / "run_scanner_validation_harness.sh" + command = [sys.executable, str(self.adapter), "measure", str(request), str(self.root / "unused.json")] + collector.write_text("#!/usr/bin/env bash\nexec " + shlex.join(command) + "\n") + + def failed_measure(*_): + wait_for_marker(marker) + raise ValueError("injected measurement failure") + + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \ + patch.object(harness, "__file__", str(self.root / "scanner_abba.py")), \ + patch.object(harness, "invoke", side_effect=failed_measure): + with self.assertRaisesRegex(ValueError, "injected measurement failure"): + harness.collect_live({"collector": {"alias": "fixture", "endpoint": "fixture", "metrics_endpoints": "fixture"}}, + {"duration_seconds": 900}, request, self.adapter) + self.assertTrue(child_released(marker), "collector parent exit did not end its telemetry child") + finally: + reap_fixture(marker) + + def test_successful_prepare_keeps_service_alive(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + self.assertEqual(harness.invoke(self.adapter, "prepare", request, 5), {"ready": True}) + self.assertFalse(child_released(marker, timeout=0), "successful prepare must preserve its service") + self.assertEqual(harness.invoke(self.adapter, "stop", request, 5), {"stopped": True}) + self.assertTrue(child_released(marker), "adapter stop must release its service") + finally: + reap_fixture(marker) + + def test_reaped_owner_never_signals_a_reused_process_group(self): + with (self.root / "owner.log").open("wb") as log: + owner = harness.OwnedCommand([sys.executable, "-c", "pass"], log) + self.assertEqual(owner.wait(5), 0) + self.assertEqual(owner.finish(), 0) + with patch.object(harness.os, "killpg", side_effect=AssertionError("released PGID must not be signalled")): + self.assertEqual(owner.finish(terminate=True), 0) + + def test_cleanup_interruption_still_kills_group_and_reaps_leader(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + original_sleep = time.sleep + interrupted = False + + def interrupt_once(delay): + nonlocal interrupted + if not interrupted: + interrupted = True + raise KeyboardInterrupt + original_sleep(delay) + + with (self.root / "interrupted.log").open("wb") as log: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + owner = harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log) + try: + wait_for_marker(marker) + with patch.object(harness.time, "sleep", side_effect=interrupt_once): + with self.assertRaises(KeyboardInterrupt): + owner.finish(terminate=True) + self.assertTrue(child_released(marker), "cleanup cancellation left its child alive") + self.assertIsNotNone(owner.process.returncode, "cleanup cancellation must reap its leader") + finally: + reap_fixture(marker) + owner.process.wait(timeout=5) + + def test_constructor_failure_after_gate_release_kills_group(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + original_write = os.write + + def release_then_fail(fd, data): + original_write(fd, data) + wait_for_marker(marker) + raise OSError("injected failure after gate release") + + try: + with (self.root / "construction.log").open("wb") as log, \ + patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \ + patch.object(harness.os, "write", side_effect=release_then_fail): + with self.assertRaisesRegex(OSError, "injected failure after gate release"): + harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log) + self.assertTrue(child_released(marker), "initialization failure left its child alive") + finally: + reap_fixture(marker) + def test_complete_synthetic_matrix_is_not_performance_evidence(self): self.assertEqual(self.run_harness(), 0) report = harness.read_json(self.root / "out/report.json") @@ -201,10 +350,9 @@ class ScannerAbbaTest(unittest.TestCase): else: harness.write_json(sample, payload) process = Mock(pid=123, wait=Mock(return_value=1 if name == "collector-exit" else 0)) - with patch.object(harness.subprocess, "Popen", return_value=process), \ + with patch.object(harness, "OwnedCommand", return_value=process), \ patch.object(harness, "invoke", return_value={"sample_count": 10}), \ - patch.object(harness.time, "monotonic", side_effect=(0, 900)), \ - patch.object(harness.os, "killpg"): + patch.object(harness.time, "monotonic", side_effect=(0, 900)): if error: with self.assertRaisesRegex(ValueError, error): harness.collect_live(prepared, {"duration_seconds": 900}, self.root / "request.json", self.adapter) @@ -212,6 +360,8 @@ class ScannerAbbaTest(unittest.TestCase): self.assertEqual(harness.collect_live(prepared, {"duration_seconds": 900}, self.root / "request.json", self.adapter), {"sample_count": 10}) + process.finish.assert_called_once_with(terminate=True) + def test_unstable_p1_work_control_is_inconclusive(self): with patch.object(harness, "SCENARIOS", ("cold-hot",)): self.assertEqual(self.run_harness("unstable-p1-control"), 3) @@ -259,6 +409,14 @@ class ScannerAbbaTest(unittest.TestCase): if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--stubborn-worker": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with Path(sys.argv[2]).open("w+") as marker: + fcntl.flock(marker, fcntl.LOCK_EX) + marker.write(str(os.getpid())) + marker.flush() + while True: + time.sleep(1) if len(sys.argv) == 4 and sys.argv[1] in ("prepare", "measure", "oracle", "stop"): sys.exit(fake_adapter()) unittest.main()