diff --git a/crates/e2e_test/src/distributed/mod.rs b/crates/e2e_test/src/distributed/mod.rs index f627b339b..a687e9356 100644 --- a/crates/e2e_test/src/distributed/mod.rs +++ b/crates/e2e_test/src/distributed/mod.rs @@ -36,6 +36,8 @@ mod s3_basic_test; mod s3_during_data_movement_test; mod scanner_heal_topology_fixture; mod scanner_heal_topology_test; +mod scanner_peer_bucket_fixture; +mod scanner_peer_bucket_test; mod site_replication_test; mod upgrade_test; mod versioning_test; diff --git a/crates/e2e_test/src/distributed/scanner_peer_bucket_fixture.rs b/crates/e2e_test/src/distributed/scanner_peer_bucket_fixture.rs new file mode 100644 index 000000000..6225e5604 --- /dev/null +++ b/crates/e2e_test/src/distributed/scanner_peer_bucket_fixture.rs @@ -0,0 +1,1026 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Real multi-set fixture for peer-owned dirty-usage certification. +//! The file barrier only observes or pauses production work. Every dirty +//! generation in these cases originates from an acknowledged S3 operation. + +use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, payload_for, sha256_hex, wait_until}; +use super::scanner_heal_topology_fixture::{assert_bucket_durability_strict, assert_process_durability_strict}; +use crate::chaos::census_object_version_on_disk; +use crate::common::admin_request; +use crate::rpc_fault_proxy::{RpcFaultProxy, ScopedAckQueue, ScopedAckScope}; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use http::Method; +use rustfs_data_usage::DataUsageInfo; +use rustfs_ecstore::api::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; +use rustfs_ecstore::api::erasure::{ReedSolomonEncoder, calc_shard_size}; +use rustfs_ecstore::api::rpc::PeerRestClient; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::{Ipv6Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use uuid::Uuid; + +const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_SCANNER_BARRIER_DIR"; +const EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_PEER_BUCKET_EVIDENCE_DIR"; +pub(super) const CYCLE_TIMEOUT: Duration = Duration::from_secs(180); +const OBSERVATION_LIMIT: usize = 512; +const OBSERVATION_BYTES_LIMIT: u64 = 1024 * 1024; + +#[derive(Clone, Debug, serde::Serialize)] +pub(super) struct ObjectReceipt { + pub bucket: String, + pub key: String, + pub version_id: String, + pub length: u64, + pub sha256: String, + pub ingress_node: usize, + pub set: usize, +} + +pub(super) struct PeerBucketRun { + pub dist: DistCluster, + pub buckets: [String; 3], + pub objects: Vec, + pub barrier: ScannerBarrier, + pub topology: Value, + pub artifacts: PathBuf, + keep_artifacts: bool, + drive_sets: BTreeMap, + proxies: Vec, +} + +impl PeerBucketRun { + pub async fn start(label: &str) -> TestResult { + Self::start_inner(label, false, false).await + } + + pub async fn start_with_ack_faults(label: &str) -> TestResult { + Self::start_inner(label, true, false).await + } + + pub async fn start_for_crash(label: &str) -> TestResult { + Self::start_inner(label, true, true).await + } + + async fn start_inner(label: &str, proxy_acks: bool, read_peer_snapshots: bool) -> TestResult { + crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks")); + let nonce = Uuid::new_v4().simple().to_string(); + let kept_root = std::env::var_os(EVIDENCE_DIR_ENV).or_else(|| std::env::var_os("RUSTFS_E2E_LOG_DIR")); + let keep_artifacts = kept_root.is_some(); + let artifacts = kept_root + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join(format!("scanner-peer-bucket-{label}-{nonce}")); + let barrier_dir = artifacts.join("scanner-barrier"); + std::fs::create_dir_all(&barrier_dir)?; + if !barrier_dir.is_absolute() { + return Err("scanner barrier directory must be absolute".into()); + } + let barrier_value = barrier_dir.to_string_lossy().into_owned(); + // An explicit cycle or start-delay value disables adaptive dirty + // wakeups. The fixture uses real mutations on the default cadence. + let env = [ + ("RUSTFS_SCANNER_ENABLED", "true"), + ("RUSTFS_SCANNER_BITROT_CYCLE_SECS", "off"), + ("RUSTFS_ERASURE_SET_DRIVE_COUNT", "4"), + ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"), + ("RUSTFS_DURABILITY_MODE", "strict"), + ("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"), + ("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"), + (BARRIER_DIR_ENV, barrier_value.as_str()), + ]; + let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &env).await?; + if read_peer_snapshots { + // Match the existing internode RPC tests without relaxing any + // authentication or replay policy. These clients only read state. + const RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret"; + let _ = rustfs_credentials::set_global_rpc_secret(RPC_SECRET.to_string()); + if rustfs_credentials::try_get_rpc_token()? != RPC_SECRET { + return Err("crash fixture could not establish its authenticated snapshot reader".into()); + } + dist.cluster.set_env("RUSTFS_RPC_SECRET", RPC_SECRET); + } + let mut proxies = Vec::new(); + if proxy_acks { + // Keep endpoint locality's shared port while separating the proxy + // and direct server listeners by loopback address family. + for node in &mut dist.cluster.nodes { + let proxy_address = node.address.parse::()?; + let direct_address = SocketAddr::from((Ipv6Addr::LOCALHOST, proxy_address.port())); + node.address = direct_address.to_string(); + node.url = format!("http://{direct_address}"); + proxies.push(RpcFaultProxy::start_on(proxy_address, direct_address).await?); + } + dist.cluster.set_env("NO_PROXY", "localhost,127.0.0.1,::1"); + dist.cluster.set_env("no_proxy", "localhost,127.0.0.1,::1"); + } + let peer_addresses = (0..4) + .map(|node| { + proxies + .get(node) + .map_or_else(|| dist.cluster.nodes[node].address.clone(), |proxy| proxy.local_addr().to_string()) + }) + .collect::>(); + // Each set contains one drive from each process. The harness defaults + // to node-major order, which would put all four drives of a set on one node. + let volumes = (0..4) + .flat_map(|drive| { + let peer_addresses = &peer_addresses; + dist.cluster + .nodes + .iter() + .enumerate() + .map(move |(index, node)| format!("http://{}{}", peer_addresses[index], node.data_dirs[drive])) + }) + .collect::>() + .join(" "); + dist.cluster.set_env("RUSTFS_VOLUMES", volumes.clone()); + dist.capture_logs_under(&artifacts.join("node-logs"))?; + dist.cluster.start().await?; + let buckets = [format!("cold-a-{nonce}"), format!("cold-b-{nonce}"), format!("hot-{nonce}")]; + let mut run = Self { + dist, + buckets, + objects: Vec::new(), + barrier: ScannerBarrier { + directory: barrier_dir, + nonce, + }, + topology: Value::Null, + artifacts, + keep_artifacts, + drive_sets: BTreeMap::new(), + proxies, + }; + if proxy_acks { + let nodes = (0..4) + .map(|node| { + Ok(json!({ + "node": node, + "proxy_address": run.proxies[node].local_addr().to_string(), + "direct_address": run.dist.cluster.nodes[node].address, + "direct_url": run.dist.cluster.nodes[node].url, + "pid": run.pid(node)?, + })) + }) + .collect::>>()?; + run.write_evidence("proxy-routing", &json!({ "nodes": nodes, "volumes": volumes }))?; + } + run.verify_topology().await?; + run.seed().await?; + let bucket_names = run.buckets.iter().map(String::as_str).collect::>(); + assert_bucket_durability_strict(&run.dist.cluster, &bucket_names).await?; + let logs = run + .dist + .cluster + .node_capture_log_paths + .iter() + .flatten() + .map(PathBuf::from) + .collect::>(); + let reported = assert_process_durability_strict(&logs)?; + if reported != 4 { + return Err(format!("strict process durability was observed on {reported}/4 nodes").into()); + } + Ok(run) + } + + async fn verify_topology(&mut self) -> TestResult { + let storage = self.admin_json(0, "/rustfs/admin/v3/storageinfo").await?; + let disks = storage["info"]["disks"].as_array().ok_or("storageinfo omitted disks")?; + if disks.len() != 16 { + return Err(format!("expected sixteen actual disks, got {}", disks.len()).into()); + } + let mut set_nodes = BTreeMap::>::new(); + let mut set_disks = BTreeMap::>::new(); + let peer_addresses = (0..4).map(|node| self.peer_address(node)).collect::>(); + for disk in disks { + if disk["pool_index"].as_u64() != Some(0) + || disk["state"].as_str().is_none_or(|state| !state.eq_ignore_ascii_case("ok")) + || disk["runtimeState"] + .as_str() + .is_none_or(|state| !state.eq_ignore_ascii_case("online")) + { + return Err(format!("disk is outside the healthy single-pool topology: {disk}").into()); + } + let set = usize::try_from(disk["set_index"].as_u64().ok_or("disk omitted set_index")?)?; + let disk_index = disk["disk_index"].as_u64().ok_or("disk omitted disk_index")?; + let endpoint = disk["endpoint"].as_str().ok_or("disk omitted endpoint")?; + let matches = self + .dist + .cluster + .nodes + .iter() + .enumerate() + .flat_map(|(node_index, node)| { + let peer_addresses = &peer_addresses; + node.data_dirs.iter().filter_map(move |path| { + (endpoint.trim_end_matches('/') == format!("http://{}{}", peer_addresses[node_index], path)) + .then_some((node_index, path)) + }) + }) + .collect::>(); + if matches.len() != 1 { + return Err(format!("storage endpoint does not identify one fixture drive: {endpoint}").into()); + } + let (node, path) = matches[0]; + if self.drive_sets.insert(path.clone(), set).is_some() + || !set_nodes.entry(set).or_default().insert(node) + || !set_disks.entry(set).or_default().insert(disk_index) + { + return Err(format!("duplicate drive, node, or disk index in set {set}").into()); + } + } + let expected = BTreeSet::from([0, 1, 2, 3]); + if set_nodes.keys().copied().collect::>() != expected + || set_nodes.values().any(|nodes| nodes != &expected) + || set_disks.values().any(|disks| disks != &BTreeSet::from([0, 1, 2, 3])) + { + return Err(format!("actual erasure sets do not each span four nodes: {set_nodes:?}, {set_disks:?}").into()); + } + self.topology = storage; + Ok(()) + } + + async fn seed(&mut self) -> TestResult { + let client = self.dist.client(0)?; + for bucket in self.buckets.clone() { + client.create_bucket().bucket(&bucket).send().await?; + enable_versioning(&client, &bucket).await?; + let mut covered = BTreeSet::new(); + for index in 0..128 { + let receipt = self.put(0, &bucket, &format!("seed-{index:03}"), 71).await?; + covered.insert(receipt.set); + if covered == BTreeSet::from([0, 1, 2, 3]) { + break; + } + } + if covered != BTreeSet::from([0, 1, 2, 3]) { + return Err(format!("bounded real PUT inventory did not cover four sets for {bucket}: {covered:?}").into()); + } + } + self.verify_objects().await + } + + pub async fn put(&mut self, node: usize, bucket: &str, key: &str, length: usize) -> TestResult { + let receipt = write_object(&self.dist.client(node)?, &self.drive_inventory()?, node, bucket, key, length).await?; + self.objects.push(receipt.clone()); + Ok(receipt) + } + + fn drive_inventory(&self) -> TestResult>> { + self.dist + .cluster + .nodes + .iter() + .map(|node| { + node.data_dirs + .iter() + .map(|drive| { + Ok(( + drive.clone(), + *self + .drive_sets + .get(drive) + .ok_or("fixture drive missing from actual topology")?, + )) + }) + .collect() + }) + .collect() + } + + pub fn start_hot_writer(&self, node: usize, bucket: &str) -> TestResult { + self.start_alternating_hot_writer(&[node], bucket) + } + + pub fn start_alternating_hot_writer(&self, nodes: &[usize], bucket: &str) -> TestResult { + if nodes.is_empty() + || nodes.len() > 2 + || nodes.iter().any(|node| *node >= 4) + || nodes.iter().collect::>().len() != nodes.len() + { + return Err("hot writer requires one or two distinct fixture ingress nodes".into()); + } + let clients = nodes + .iter() + .map(|node| self.dist.client(*node).map(|client| (*node, client))) + .collect::>>()?; + let drives = self.drive_inventory()?; + let bucket = bucket.to_string(); + let (stop, mut stopped) = tokio::sync::watch::channel(false); + let (progress, completed) = tokio::sync::watch::channel(0); + let task = tokio::spawn(async move { + let mut receipts = Vec::new(); + for index in 0..256 { + if *stopped.borrow() { + return Ok(receipts); + } + let (node, client) = &clients[index % clients.len()]; + receipts.push(write_object(client, &drives, *node, &bucket, &format!("hot-stream-{index:03}"), 83).await?); + let _ = progress.send(receipts.len()); + tokio::select! { + _ = stopped.changed() => {}, + _ = tokio::time::sleep(Duration::from_millis(100)) => {}, + } + } + Err("bounded hot writer exhausted 256 actual PUTs before certification completed".into()) + }); + Ok(HotWriter { + stop, + completed, + task: Some(task), + }) + } + + pub fn peer_address(&self, node: usize) -> String { + self.proxies + .get(node) + .map_or_else(|| self.dist.cluster.nodes[node].address.clone(), |proxy| proxy.local_addr().to_string()) + } + + pub fn scoped_ack_queue( + &self, + node: usize, + owner: &Value, + buckets: BTreeSet, + capacity: usize, + ) -> TestResult { + let proxy = self.proxies.get(node).ok_or("fixture did not enable the scoped ACK proxy")?; + Ok(proxy.scoped_ack_queue( + ScopedAckScope { + nonce: self.barrier.nonce.clone(), + owner_id: owner["owner_id"] + .as_str() + .ok_or("scoped ACK receiver omitted owner")? + .to_string(), + instance_id: owner["instance_id"] + .as_str() + .ok_or("scoped ACK receiver omitted instance")? + .to_string(), + buckets, + }, + capacity, + )?) + } + + pub async fn verify_objects(&self) -> TestResult { + for client in self.dist.clients()? { + for bucket in &self.buckets { + let listed = client.list_object_versions().bucket(bucket).send().await?; + if listed.is_truncated() == Some(true) || !listed.delete_markers().is_empty() { + return Err("fixture version listing is truncated or contains an unexpected delete marker".into()); + } + let expected = self + .objects + .iter() + .filter(|receipt| receipt.bucket == *bucket) + .map(|receipt| (receipt.key.as_str(), receipt.version_id.as_str(), receipt.length as i64)) + .collect::>(); + let actual = listed + .versions() + .iter() + .map(|version| { + Ok(( + version.key().ok_or("version listing omitted key")?, + version.version_id().ok_or("version listing omitted version ID")?, + version.size().ok_or("version listing omitted size")?, + )) + }) + .collect::>>()?; + if actual != expected { + return Err(format!("version listing differs from acknowledged S3 receipts for {bucket}").into()); + } + } + for receipt in &self.objects { + let output = client + .get_object() + .bucket(&receipt.bucket) + .key(&receipt.key) + .version_id(&receipt.version_id) + .send() + .await?; + if output.version_id() != Some(receipt.version_id.as_str()) { + return Err(format!("GET returned a different version for {}", receipt.key).into()); + } + let body = output.body.collect().await?.into_bytes(); + if body.len() as u64 != receipt.length || sha256_hex(&body) != receipt.sha256 { + return Err(format!("GET bytes differ from acknowledged S3 receipt for {}", receipt.key).into()); + } + } + } + Ok(()) + } + + pub async fn admin_json(&self, node: usize, path: &str) -> TestResult { + let (status, body) = admin_request( + &self.dist.cluster.nodes[node].url, + Method::GET, + path, + None, + &self.dist.cluster.access_key, + &self.dist.cluster.secret_key, + ) + .await?; + if !status.is_success() { + return Err(format!("node {node} GET {path} returned {status}: {body}").into()); + } + Ok(serde_json::from_str(&body)?) + } + + pub async fn leader(&self) -> TestResult { + let mut leaders = Vec::new(); + for node in 0..self.dist.cluster.nodes.len() { + let status = self.admin_json(node, "/rustfs/admin/v3/scanner/status").await?; + if status["metrics"]["leader_lock_held_by_this_process"].as_bool() == Some(true) { + leaders.push(node); + } + } + if leaders.len() != 1 { + return Err(format!("expected one observed scanner coordinator, got {leaders:?}").into()); + } + Ok(leaders[0]) + } + + pub async fn assert_default_cadence(&self) -> TestResult> { + let mut configs = Vec::new(); + for node in 0..self.dist.cluster.nodes.len() { + let status = self.admin_json(node, "/rustfs/admin/v3/scanner/status").await?; + let config = &status["runtime_config"]; + if config["cycle_interval_seconds"]["source"].as_str() != Some("default") + || config["start_delay_seconds"]["source"].as_str() != Some("default") + || !config["start_delay_seconds"]["value"].is_null() + { + return Err( + format!("node {node} must use the default scanner cadence without a start-delay override: {config}").into(), + ); + } + configs.push(config.clone()); + } + Ok(configs) + } + + pub fn pid(&self, node: usize) -> TestResult { + self.dist.cluster.nodes[node] + .process + .as_ref() + .map(std::process::Child::id) + .ok_or_else(|| format!("node {node} is not running").into()) + } + + pub async fn dirty_snapshot(&self, node: usize) -> TestResult { + let address = self.peer_address(node).parse::()?; + let client = PeerRestClient::new( + rustfs_utils::net::XHost { + name: address.ip().to_string(), + port: address.port(), + is_port_set: true, + }, + format!("http://{address}"), + ); + // The production client verifies the challenge-bound response proof, + // protocol, owner, incarnation, ordering and generation bounds. + let snapshot = client.scanner_dirty_usage_snapshot().await?; + Ok(json!({ + "node": node, "pid": self.pid(node)?, "host": address.to_string(), + "owner_id": snapshot.owner_id, "instance_id": snapshot.instance_id, + "generation": snapshot.generation, "pending_bucket_count": snapshot.pending_bucket_count, + "protocol_version": snapshot.protocol_version, "complete": snapshot.complete, + "buckets": snapshot.buckets.into_iter().map(|(bucket, entry)| json!({ + "bucket": bucket, "bucket_incarnation": entry.bucket_incarnation, "generation": entry.generation, + })).collect::>(), + })) + } + + pub fn drive_identity(&self, node: usize) -> TestResult { + self.dist.cluster.nodes[node] + .data_dirs + .iter() + .map(|drive| { + let path = Path::new(drive).join(RUSTFS_META_BUCKET).join("format.json"); + let bytes = read_bounded_file(&path)?; + Ok(json!({"drive": drive, "set": self.drive_sets[drive], "format_sha256": sha256_hex(&bytes), + "format": serde_json::from_slice::(&bytes)?})) + }) + .collect::>>() + .map(Value::Array) + } + + pub fn executable_identity(&self) -> TestResult { + use sha2::{Digest, Sha256}; + use std::io::Read; + + let path = crate::common::rustfs_binary_path_with_features(Some("e2e-test-hooks")); + let mut file = std::fs::File::open(&path)?; + let size = file.metadata()?.len(); + let mut digest = Sha256::new(); + let mut buffer = [0; 64 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + Ok(json!({"path": path, "size": size, + "sha256": hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower)})) + } + + pub fn kill_at(&mut self, node: usize, gate: &Value) -> TestResult { + if !cfg!(unix) { + return Err("scanner crash windows require Unix SIGKILL semantics".into()); + } + let pid = self.pid(node)?; + if gate["armed"] != true || gate["pid"].as_u64() != Some(u64::from(pid)) { + return Err("crash target is not the actual process held at the requested barrier".into()); + } + let drives = self.drive_identity(node)?; + let process = self.dist.cluster.nodes[node] + .process + .as_mut() + .ok_or("crash target process disappeared")?; + if process.try_wait()?.is_some() { + return Err("crash target exited before SIGKILL".into()); + } + process.kill()?; + let status = process.wait()?; + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if status.signal() != Some(9) { + return Err(format!("scanner crash did not terminate with SIGKILL: {status}").into()); + } + } + self.dist.cluster.nodes[node].process = None; + Ok( + json!({"node": node, "pid": pid, "signal": 9, "wait_status": status.to_string(), "gate": gate, + "drives": drives, "direct_address": self.dist.cluster.nodes[node].address, "peer_address": self.peer_address(node)}), + ) + } + + pub async fn restart_after_crash(&mut self, crash: &Value, old_snapshot: &Value) -> TestResult { + let node = usize::try_from(crash["node"].as_u64().ok_or("crash receipt omitted node")?)?; + if self.drive_identity(node)? != crash["drives"] { + return Err("crashed node drive paths or format identity changed before restart".into()); + } + self.dist.cluster.start_node(node).await?; + super::harness::wait_for_ready(&self.dist.cluster).await?; + let mut snapshot_attempts = 1; + let snapshot = match self.dirty_snapshot(node).await { + Ok(snapshot) => snapshot, + Err(error) => { + let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(error.as_ref()); + let mut epoch_changed = false; + for _ in 0..8 { + let Some(current) = cause else { break }; + if let Some(status) = current.downcast_ref::() { + epoch_changed = status.code() == tonic::Code::Unavailable && status.message() == "RPC boot epoch changed"; + break; + } + cause = current.source(); + } + if !epoch_changed { + return Err(error); + } + self.write_evidence( + "restart-rpc-rejection", + &json!({"node": node, "pid": self.pid(node)?, "attempt": 1, + "code": "Unavailable", "message": "RPC boot epoch changed"}), + )?; + // The rejected read refreshes the authenticated peer epoch. + // A fresh client must then validate one new snapshot normally. + snapshot_attempts = 2; + self.dirty_snapshot(node).await? + } + }; + if snapshot["pid"] == crash["pid"] + || snapshot["instance_id"] == old_snapshot["instance_id"] + || snapshot["owner_id"] != old_snapshot["owner_id"] + || self.drive_identity(node)? != crash["drives"] + || self.dist.cluster.nodes[node].address != crash["direct_address"] + || self.peer_address(node) != crash["peer_address"] + { + return Err("restart did not replace the process instance on unchanged owner, endpoints and drives".into()); + } + let names = self.buckets.iter().map(String::as_str).collect::>(); + assert_bucket_durability_strict(&self.dist.cluster, &names).await?; + self.assert_default_cadence().await?; + self.write_evidence("restart-rpc-handshake", &json!({"attempts": snapshot_attempts, "snapshot": snapshot}))?; + Ok(snapshot) + } + + /// Decode only the bounded single-stripe root used by this fixture. Every + /// present replica must agree; this never elects a convenient old revision. + pub fn authoritative_root(&self) -> TestResult { + let key = format!("{BUCKET_META_PREFIX}/{}", rustfs_data_usage::DATA_USAGE_OBJECT_NAME); + let mut common = None; + let mut shards = BTreeMap::new(); + let mut locations = Vec::new(); + let mut nodes = BTreeSet::new(); + let mut sets = BTreeSet::new(); + for (node, process) in self.dist.cluster.nodes.iter().enumerate() { + for drive in &process.data_dirs { + let object = Path::new(drive).join(RUSTFS_META_BUCKET).join(&key); + let path = object.join("xl.meta"); + if !path.try_exists()? { + continue; + } + let metadata = read_bounded_file(&path)?; + let info = + rustfs_filemeta::FileMeta::load(&metadata)?.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?; + info.validate_for_metadata_read()?; + let size = usize::try_from(info.size)?; + if info.deleted + || info.uses_legacy_checksum + || size == 0 + || size > 1024 * 1024 + || size > info.erasure.block_size + || info.parts.len() != 1 + || info.parts[0].size != size + || info.erasure.data_blocks != 2 + || info.erasure.parity_blocks != 2 + || !(1..=4).contains(&info.erasure.index) + || info.erasure.distribution.iter().copied().collect::>() != BTreeSet::from([1, 2, 3, 4]) + { + return Err( + format!("authoritative root is not the fixture's bounded modern EC:2 single stripe: {path:?}").into(), + ); + } + let version = info + .get_etag() + .filter(|value| !value.is_empty()) + .ok_or("root metadata omitted ETag")?; + let revision = json!({"root_version": version, "size": size, "version_id": info.version_id, + "mod_time": info.mod_time, "data_dir": info.data_dir, "distribution": info.erasure.distribution, + "block_size": info.erasure.block_size, "algorithm": info.erasure.algorithm}); + if common.as_ref().is_some_and(|previous| previous != &revision) { + return Err("authoritative root metadata disagrees across physical replicas".into()); + } + common = Some(revision); + let framed = if info.inline_data() { + info.data.as_ref().ok_or("inline root omitted shard bytes")?.to_vec() + } else { + let part = object + .join(info.data_dir.ok_or("external root omitted data directory")?.to_string()) + .join(format!("part.{}", info.parts[0].number)); + read_bounded_file(&part)? + }; + let checksum = info.erasure.get_checksum_info(info.parts[0].number); + let hash_size = checksum.algorithm.size(); + let shard_size = calc_shard_size(size, info.erasure.data_blocks); + if hash_size == 0 + || framed.len() != hash_size + shard_size + || checksum.algorithm.hash_encode(&framed[hash_size..]).as_ref() != &framed[..hash_size] + { + return Err(format!("authoritative root shard failed its actual bitrot framing: {path:?}").into()); + } + if shards.insert(info.erasure.index, framed[hash_size..].to_vec()).is_some() || !nodes.insert(node) { + return Err("authoritative root repeated an erasure index or node".into()); + } + sets.insert(self.drive_sets[drive]); + locations.push(json!({"node": node, "drive": drive, "index": info.erasure.index, + "xl_meta_sha256": sha256_hex(&metadata), "xl_meta_hex": hex_simd::encode_to_string(&metadata, hex_simd::AsciiCase::Lower), + "shard_sha256": sha256_hex(&framed), "shard_frame_hex": hex_simd::encode_to_string(&framed, hex_simd::AsciiCase::Lower)})); + } + } + if nodes != BTreeSet::from([0, 1, 2, 3]) || shards.len() != 4 || sets.len() != 1 { + return Err("authoritative root lacks four agreeing replicas in one actual erasure set".into()); + } + let mut root = common.ok_or("authoritative root metadata is absent")?; + let size = usize::try_from(root["size"].as_u64().ok_or("root size is absent")?)?; + let codec = ReedSolomonEncoder::new(2, 2)?; + let mut data = None; + let mut read_quorums = Vec::new(); + for first in 1..=4 { + for second in first + 1..=4 { + let mut decoded = (1..=4) + .map(|index| (index == first || index == second).then(|| shards[&index].clone())) + .collect::>(); + codec.reconstruct_data(&mut decoded)?; + let mut payload = decoded[0].as_ref().ok_or("root data shard 1 was not reconstructed")?.clone(); + payload.extend_from_slice(decoded[1].as_ref().ok_or("root data shard 2 was not reconstructed")?); + payload.truncate(size); + if data.as_ref().is_some_and(|previous| previous != &payload) { + return Err("authoritative root payload differs between actual erasure read quorums".into()); + } + data = Some(payload); + read_quorums.push([first, second]); + } + } + let data = data.ok_or("authoritative root has no decoded read quorum")?; + let usage: DataUsageInfo = serde_json::from_slice(&data)?; + if !usage.is_complete_bucket_usage_snapshot() || usage.usage_snapshot_observational { + return Err("physical authoritative root is not a complete authoritative usage snapshot".into()); + } + root["root_sha256"] = json!(sha256_hex(&data)); + root["payload_hex"] = json!(hex_simd::encode_to_string(&data, hex_simd::AsciiCase::Lower)); + root["usage"] = serde_json::to_value(usage)?; + root["shards"] = json!(locations); + root["read_quorums"] = json!(read_quorums); + Ok(root) + } + + pub fn usage_matches(&self, usage: &DataUsageInfo, buckets: &[&str]) -> bool { + buckets.iter().all(|bucket| { + let objects = self + .objects + .iter() + .filter(|receipt| receipt.bucket == *bucket) + .collect::>(); + usage.buckets_usage.get(*bucket).is_some_and(|actual| { + actual.objects_count == objects.iter().map(|receipt| &receipt.key).collect::>().len() as u64 + && actual.versions_count == objects.len() as u64 + && actual.delete_markers_count == 0 + && actual.size == objects.iter().map(|receipt| receipt.length).sum::() + }) + }) + } + + pub async fn quiet_baseline(&self) -> TestResult { + let names = self.buckets.iter().map(String::as_str).collect::>(); + wait_until( + CYCLE_TIMEOUT, + || async { + let Ok(leader) = self.leader().await else { return Ok(false) }; + let usage: DataUsageInfo = + serde_json::from_value(self.admin_json(leader, "/rustfs/admin/v3/datausageinfo").await?)?; + Ok(usage.is_complete_bucket_usage_snapshot() + && usage.usage_snapshot_converged == Some(true) + && usage.usage_snapshot_pending_buckets.is_empty() + && usage.scanner_epoch.is_some() + && usage.scanner_cycle.is_some() + && self.usage_matches(&usage, &names)) + }, + "quiet authoritative peer-bucket baseline", + ) + .await?; + let leader = self.leader().await?; + Ok(serde_json::from_value(self.admin_json(leader, "/rustfs/admin/v3/datausageinfo").await?)?) + } + + pub fn write_evidence(&self, name: &str, evidence: &Value) -> TestResult { + std::fs::write(self.artifacts.join(format!("{name}.json")), serde_json::to_vec_pretty(evidence)?)?; + Ok(()) + } +} + +fn read_bounded_file(path: &Path) -> TestResult> { + use std::io::Read; + + let file = std::fs::File::open(path)?; + if !file.metadata()?.is_file() || file.metadata()?.len() > OBSERVATION_BYTES_LIMIT { + return Err(format!("fixture evidence file exceeds its regular-file bound: {path:?}").into()); + } + let mut bytes = Vec::new(); + file.take(OBSERVATION_BYTES_LIMIT + 1).read_to_end(&mut bytes)?; + if u64::try_from(bytes.len())? > OBSERVATION_BYTES_LIMIT { + return Err(format!("fixture evidence file grew past its byte bound: {path:?}").into()); + } + Ok(bytes) +} + +async fn write_object( + client: &Client, + drives: &[Vec<(String, usize)>], + node: usize, + bucket: &str, + key: &str, + length: usize, +) -> TestResult { + let body = payload_for(key, length); + let output = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(body.clone())) + .send() + .await?; + let version = output.version_id().ok_or("versioned fixture PUT returned no version ID")?; + if Uuid::parse_str(version)?.is_nil() { + return Err("versioned fixture PUT returned a nil version ID".into()); + } + let mut object_sets = BTreeSet::new(); + for process in drives { + let mut complete = 0; + for (drive, set) in process { + let census = census_object_version_on_disk(Path::new(drive), bucket, key, Some(version))?; + if census.has_xl_meta { + if !census.is_complete() || (census.data_blocks, census.parity_blocks) != (Some(2), Some(2)) { + return Err(format!("acknowledged PUT has an incomplete EC:2 shard on {drive}: {census:?}").into()); + } + complete += 1; + object_sets.insert(*set); + } + } + if complete != 1 { + return Err(format!("acknowledged PUT must have one complete shard per node, got {complete}").into()); + } + } + if object_sets.len() != 1 { + return Err(format!("object shards crossed erasure sets: {object_sets:?}").into()); + } + Ok(ObjectReceipt { + bucket: bucket.to_string(), + key: key.to_string(), + version_id: version.to_string(), + length: u64::try_from(length)?, + sha256: sha256_hex(&body), + ingress_node: node, + set: *object_sets.first().ok_or("object has no physical set")?, + }) +} + +pub(super) struct HotWriter { + stop: tokio::sync::watch::Sender, + completed: tokio::sync::watch::Receiver, + task: Option>>>, +} + +impl HotWriter { + pub fn completed(&self) -> usize { + *self.completed.borrow() + } + + pub async fn wait_for_completed(&mut self, minimum: usize) -> TestResult { + tokio::time::timeout(Duration::from_secs(30), async { + while self.completed() < minimum { + self.completed + .changed() + .await + .map_err(|_| "hot writer stopped before the required S3 receipt")?; + } + Ok::<_, Box>(()) + }) + .await??; + Ok(()) + } + + pub async fn finish(mut self) -> TestResult> { + let _ = self.stop.send(true); + self.task.take().ok_or("hot writer already joined")?.await? + } +} + +impl Drop for HotWriter { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl Drop for PeerBucketRun { + fn drop(&mut self) { + self.dist.cluster.stop(); + if !self.keep_artifacts { + let _ = std::fs::remove_dir_all(&self.artifacts); + } + } +} + +pub(super) struct ScannerBarrier { + directory: PathBuf, + nonce: String, +} + +impl ScannerBarrier { + pub fn start_session(&self, buckets: &[String]) -> TestResult { + self.write_json("session.json", &json!({"nonce": self.nonce, "buckets": buckets})) + } + + pub fn stop_session(&self) -> TestResult { + self.disarm()?; + std::fs::remove_file(self.directory.join("session.json"))?; + Ok(()) + } + + pub fn arm(&self, stage: &str, pid: u32, identity: Option<(u64, u64)>) -> TestResult { + let release = self.directory.join(format!("release.{}.{stage}.{pid}", self.nonce)); + if release.exists() { + std::fs::remove_file(release)?; + } + let mut arm = json!({"nonce": self.nonce, "stage": stage, "pid": pid}); + if let Some((epoch, cycle)) = identity { + arm["epoch"] = json!(epoch); + arm["cycle"] = json!(cycle); + } + self.write_json("arm.json", &arm) + } + + pub fn release(&self, stage: &str, pid: u32) -> TestResult { + std::fs::write(self.directory.join(format!("release.{}.{stage}.{pid}", self.nonce)), b"release")?; + Ok(()) + } + + pub fn arm_receiver(&self, stage: &str, pid: u32, peer: &Value) -> TestResult { + self.arm(stage, pid, None)?; + self.write_json( + "arm.json", + &json!({"nonce": self.nonce, "stage": stage, "pid": pid, + "owner_id": peer["owner_id"], "instance_id": peer["instance_id"]}), + ) + } + + pub fn disarm(&self) -> TestResult { + match std::fs::remove_file(self.directory.join("arm.json")) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } + + fn write_json(&self, name: &str, value: &Value) -> TestResult { + let temporary = self.directory.join(format!("{name}.tmp")); + std::fs::write(&temporary, serde_json::to_vec(value)?)?; + std::fs::rename(temporary, self.directory.join(name))?; + Ok(()) + } + + pub fn observations(&self, reached: bool) -> TestResult> { + let prefix = if reached { "reached." } else { "observed." }; + let mut observations = Vec::new(); + for entry in std::fs::read_dir(&self.directory)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with(&format!("{prefix}{}.", self.nonce)) || !name.ends_with(".json") { + continue; + } + if !entry.file_type()?.is_file() || entry.metadata()?.len() > OBSERVATION_BYTES_LIMIT { + return Err("scanner observation is not a bounded regular file".into()); + } + let value: Value = serde_json::from_slice(&std::fs::read(entry.path())?)?; + if value["nonce"].as_str() != Some(self.nonce.as_str()) { + return Err("scanner observation nonce differs from the fixture session".into()); + } + observations.push(value); + if observations.len() > OBSERVATION_LIMIT { + return Err("scanner observation count exceeded the bounded fixture limit".into()); + } + } + observations.sort_by_key(|value| (value["pid"].as_u64(), value["sequence"].as_u64())); + Ok(observations) + } + + pub async fn wait_stage(&self, stage: &str, pid: u32, identity: Option<(u64, u64)>, reached: bool) -> TestResult { + self.wait_stage_at_or_after(stage, pid, identity, reached, 0).await + } + + pub async fn wait_stage_after( + &self, + stage: &str, + pid: u32, + identity: Option<(u64, u64)>, + reached: bool, + after_sequence: u64, + ) -> TestResult { + self.wait_stage_at_or_after(stage, pid, identity, reached, after_sequence.saturating_add(1)) + .await + } + + async fn wait_stage_at_or_after( + &self, + stage: &str, + pid: u32, + identity: Option<(u64, u64)>, + reached: bool, + minimum_sequence: u64, + ) -> TestResult { + let select = || -> TestResult> { + Ok(self.observations(reached)?.into_iter().find(|value| { + value["stage"].as_str() == Some(stage) + && value["pid"].as_u64() == Some(u64::from(pid)) + && (!reached || value["armed"].as_bool() == Some(true)) + && value["sequence"] + .as_u64() + .is_some_and(|sequence| sequence >= minimum_sequence) + && identity.is_none_or(|(epoch, cycle)| { + value["detail"]["epoch"].as_u64() == Some(epoch) && value["detail"]["cycle"].as_u64() == Some(cycle) + }) + })) + }; + wait_until( + CYCLE_TIMEOUT, + || async { Ok(select()?.is_some()) }, + &format!("scanner stage {stage} on pid {pid}"), + ) + .await?; + select()?.ok_or_else(|| format!("scanner stage {stage} disappeared after observation").into()) + } +} diff --git a/crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs b/crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs new file mode 100644 index 000000000..6241b82b1 --- /dev/null +++ b/crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs @@ -0,0 +1,1784 @@ +// Copyright 2026 RustFS Team +// SPDX-License-Identifier: Apache-2.0 + +use super::harness::{TestResult, wait_until}; +use super::scanner_peer_bucket_fixture::{CYCLE_TIMEOUT, PeerBucketRun}; +use crate::common::init_logging; +use crate::rpc_fault_proxy::ScopedAckFault; +use rustfs_data_usage::DataUsageInfo; +use serde_json::{Value, json}; +use std::collections::{BTreeMap, BTreeSet}; + +fn identity(observation: &Value) -> TestResult<(u64, u64)> { + Ok(( + observation["detail"]["epoch"] + .as_u64() + .ok_or("scanner observation omitted epoch")?, + observation["detail"]["cycle"] + .as_u64() + .ok_or("scanner observation omitted cycle")?, + )) +} + +fn sequence(observation: &Value) -> TestResult { + observation["sequence"] + .as_u64() + .ok_or_else(|| "scanner observation omitted sequence".into()) +} + +fn names(value: &Value) -> TestResult> { + value + .as_array() + .ok_or_else(|| "scanner bucket observation is not an array".into()) + .and_then(|items| { + items + .iter() + .map(|value| { + value + .as_str() + .map(str::to_string) + .ok_or_else(|| "scanner bucket name is not a string".into()) + }) + .collect() + }) +} + +fn peer_for_address<'a>(observation: &'a Value, address: &str) -> TestResult<&'a Value> { + let peers = observation["detail"]["peers"] + .as_array() + .ok_or("scanner snapshot omitted actual peer maps")?; + let matching = peers + .iter() + .filter(|peer| { + peer["host"].as_str().is_some_and(|host| { + host.trim_start_matches("http://") + .trim_start_matches("https://") + .trim_end_matches('/') + == address + }) + }) + .collect::>(); + if matching.len() != 1 { + return Err(format!("actual snapshot must identify the remote ingress process once: {address}, {peers:?}").into()); + } + Ok(matching[0]) +} + +fn tuples(entries: &Value) -> TestResult> { + let mut result = BTreeMap::new(); + for entry in entries.as_array().ok_or("scoped observation omitted entries")? { + let bucket = entry["bucket"].as_str().ok_or("scoped tuple omitted bucket")?; + let incarnation = entry["bucket_incarnation"] + .as_str() + .ok_or("scoped tuple omitted bucket incarnation")?; + if uuid::Uuid::parse_str(incarnation)?.is_nil() { + return Err("scoped tuple contains a nil incarnation".into()); + } + let generation = entry["generation"] + .as_u64() + .filter(|value| *value > 0) + .ok_or("scoped tuple omitted generation")?; + if result + .insert(bucket.to_string(), (incarnation.to_string(), generation)) + .is_some() + { + return Err("scoped observation contains duplicate buckets".into()); + } + } + Ok(result) +} + +fn assert_snapshot_window(observation: &Value, peer: &Value) -> TestResult { + let host = peer["host"].as_str().ok_or("snapshot peer omitted host")?; + let before = &observation["detail"]["activity_before"][host]; + let after = &observation["detail"]["activity_after"][host]; + let generation = peer["generation"].as_u64().ok_or("snapshot peer omitted generation")?; + let lower = before["dirty_usage_generation"] + .as_u64() + .ok_or("leading peer activity omitted generation")?; + let upper = after["dirty_usage_generation"] + .as_u64() + .ok_or("trailing peer activity omitted generation")?; + if peer["complete"].as_bool() != Some(true) + || peer["instance_id"] != before["instance_id"] + || peer["instance_id"] != after["instance_id"] + || lower >= generation + || generation > upper + { + return Err( + format!("real remote write did not advance a complete snapshot inside one process/activity window: {peer}").into(), + ); + } + Ok(()) +} + +fn peer_identities(peers: &Value) -> TestResult> { + let mut identities = BTreeSet::new(); + for peer in peers.as_array().ok_or("peer identities are not an array")? { + let host = peer["host"].as_str().ok_or("peer identity omitted host")?; + let owner = peer["owner_id"].as_str().ok_or("peer identity omitted owner")?; + let instance = peer["instance_id"].as_str().ok_or("peer identity omitted instance")?; + if uuid::Uuid::parse_str(owner)?.is_nil() || instance.is_empty() { + return Err("peer identity contains an empty owner or process instance".into()); + } + if !identities.insert((host.to_string(), owner.to_string(), instance.to_string())) { + return Err("peer identities contain a duplicate".into()); + } + } + if identities.len() != 3 + || identities + .iter() + .map(|(_, _, instance)| instance) + .collect::>() + .len() + != 3 + { + return Err("the four-process fixture must observe exactly three distinct remote process instances".into()); + } + Ok(identities) +} + +fn assert_scope(candidate: &Value, expected: &BTreeSet, incarnations: &BTreeMap) -> TestResult { + assert_completed_scope(candidate, expected, incarnations, false) +} + +fn assert_completed_scope( + candidate: &Value, + expected: &BTreeSet, + incarnations: &BTreeMap, + allow_full_scan: bool, +) -> TestResult { + let detail = &candidate["detail"]; + let selected = &detail["selected_scope"]; + let expected_scope = (allow_full_scan && selected["kind"] == "all") + || (selected["kind"] == "buckets" && names(&selected["buckets"])? == *expected); + if detail["scan_mode"].as_str() != Some("normal") + || detail["status"].as_str() != Some("complete") + || !expected_scope + || names(&detail["scanned_buckets"])? != *expected + { + return Err(format!("cycle did not complete the exact Normal bucket scope: {detail}").into()); + } + let coverage = detail["set_coverage"] + .as_array() + .ok_or("candidate omitted actual set completion evidence")?; + let mut sets = BTreeSet::new(); + for set in coverage { + if set["pool"].as_u64() != Some(0) || names(&set["buckets"])? != *expected { + return Err(format!("set completed a different bucket scope: {set}").into()); + } + for bucket in expected { + if set["bucket_incarnations"][bucket].as_str() != incarnations.get(bucket).map(|entry| entry.0.as_str()) { + return Err(format!("actual set completion did not bind the same bucket incarnation for {bucket}: {set}").into()); + } + } + if !sets.insert(set["set"].as_u64().ok_or("set completion omitted set index")?) { + return Err("candidate repeated a set completion".into()); + } + } + if sets != BTreeSet::from([0, 1, 2, 3]) { + return Err(format!("candidate lacks actual completion from all four sets: {sets:?}").into()); + } + Ok(()) +} + +fn receiver_observation(run: &PeerBucketRun, stage: &str, pid: u32, expected_ack: &Value) -> TestResult> { + receiver_observation_after(run, stage, pid, expected_ack, None) +} + +fn receiver_observation_after( + run: &PeerBucketRun, + stage: &str, + pid: u32, + expected_ack: &Value, + after_sequence: Option, +) -> TestResult> { + select_receiver_observation(run.barrier.observations(false)?, stage, pid, expected_ack, after_sequence) +} + +fn select_receiver_observation( + observations: Vec, + stage: &str, + pid: u32, + expected_ack: &Value, + after_sequence: Option, +) -> TestResult> { + let expected_entries = tuples(&expected_ack["entries"])?; + for observation in observations { + let detail = &observation["detail"]; + if observation["stage"].as_str() == Some(stage) + && observation["pid"].as_u64() == Some(u64::from(pid)) + && match after_sequence { + Some(previous) => sequence(&observation)? > previous, + None => true, + } + && detail["owner_id"] == expected_ack["owner_id"] + && detail["instance_id"] == expected_ack["instance_id"] + && tuples(&detail["entries"])? == expected_entries + { + return Ok(Some(observation)); + } + } + Ok(None) +} + +#[test] +fn scanner_peer_bucket_receiver_observation_keeps_zero_and_excludes_prior_sequences() -> TestResult { + let ack = json!({ + "owner_id": "00000000-0000-0000-0000-000000000001", + "instance_id": "a".repeat(32), + "entries": [{"bucket": "cold-a", "bucket_incarnation": "00000000-0000-0000-0000-000000000002", "generation": 7}], + }); + let observation = |stage: &str, sequence: u64| { + json!({"stage": stage, "pid": 17, "sequence": sequence, "detail": { + "owner_id": ack["owner_id"], "instance_id": ack["instance_id"], "entries": ack["entries"], + }}) + }; + let before = observation("ack_before_apply", 0); + let after = observation("ack_after_apply", 1); + let later_before = observation("ack_before_apply", 2); + let later_after = observation("ack_after_apply", 3); + let observations = vec![before.clone(), after.clone(), later_before.clone(), later_after.clone()]; + assert_eq!( + select_receiver_observation(observations.clone(), "ack_before_apply", 17, &ack, None)?, + Some(before), + "the first receiver event may have sequence zero" + ); + assert_eq!( + select_receiver_observation(observations.clone(), "ack_after_apply", 17, &ack, None)?, + Some(after), + "the initial apply pair must retain its real 0/1 events" + ); + assert_eq!( + select_receiver_observation(observations.clone(), "ack_before_apply", 17, &ack, Some(0))?, + Some(later_before), + "an actual sequence-zero cursor must exclude the earlier matching event" + ); + assert_eq!( + select_receiver_observation(observations.clone(), "ack_after_apply", 17, &ack, Some(1))?, + Some(later_after), + "a later logical duplicate must not reuse the first apply observation" + ); + assert_eq!( + select_receiver_observation(observations, "ack_after_apply", 17, &ack, Some(3))?, + None, + "an exclusive cursor must not select its own observation" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_hot_window_certifies_cold_and_scopes_next_normal_cycle() -> TestResult { + hot_window(None).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_alternating_producers_retry_cold_after_scoped_ack_request_loss() -> TestResult { + hot_window(Some(ScopedAckFault::RequestLoss)).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_alternating_producers_preserve_hot_after_scoped_ack_response_loss() -> TestResult { + hot_window(Some(ScopedAckFault::ResponseLoss)).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_logical_duplicate_scoped_acks_clear_cold_once() -> TestResult { + reordered_scoped_acks(false).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_reordered_scoped_ack_preserves_newest_cold_generation() -> TestResult { + reordered_scoped_acks(true).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_root_before_publish_sigkill_preserves_old_root_and_dirty_scope() -> TestResult { + crash_window("root_before_publish").await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_root_durable_sigkill_preserves_committed_root_without_ack() -> TestResult { + crash_window("root_durable").await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_ack_before_apply_sigkill_recovers_unapplied_scope() -> TestResult { + crash_window("ack_before_apply").await +} + +#[tokio::test(flavor = "multi_thread")] +async fn scanner_peer_bucket_ack_after_apply_sigkill_recovers_without_response() -> TestResult { + crash_window("ack_after_apply").await +} + +fn assert_root_binding(candidate: &Value, root: &Value) -> TestResult { + for field in ["candidate_sha256", "coverage_sha256"] { + let digest = candidate["detail"][field].as_str().filter(|value| value.len() == 64); + if digest.is_none() || root["detail"][field].as_str() != digest { + return Err(format!("durable root did not bind the actual candidate {field}").into()); + } + } + if root["detail"]["root_sha256"].as_str().is_none_or(|value| value.len() != 64) + || root["detail"]["root_version"].is_null() + || !matches!(root["detail"]["outcome"].as_str(), Some("saved" | "already_durable")) + { + return Err(format!("root omitted its verified durable publication receipt: {root}").into()); + } + Ok(()) +} + +fn scoped_ack_for_peer<'a>(root: &Value, send: &'a Value, peer: &Value) -> TestResult<&'a Value> { + if sequence(root)? >= sequence(send)? + || identity(root)? != identity(send)? + || send["detail"]["candidate_sha256"] != root["detail"]["candidate_sha256"] + || send["detail"]["coverage_sha256"] != root["detail"]["coverage_sha256"] + { + return Err("scoped ACK did not follow the same candidate's verified durable root".into()); + } + send["detail"]["acks"] + .as_array() + .ok_or("sender omitted actual acknowledgements")? + .iter() + .find(|ack| ack["kind"] == "scoped" && ack["owner_id"] == peer["owner_id"] && ack["instance_id"] == peer["instance_id"]) + .ok_or_else(|| "sender omitted the actual producer's scoped ACK".into()) +} + +fn assert_retained_ack(receipt: &Value, ack: &Value, receiver: &str) -> TestResult { + if receipt["request"]["owner_id"] != ack["owner_id"] + || receipt["request"]["instance_id"] != ack["instance_id"] + || receipt["request"]["probe_only"] != false + || tuples(&receipt["request"]["entries"])? != tuples(&ack["entries"])? + || receipt["receiver"].as_str() != Some(receiver) + || receipt["forwarded"] != false + || receipt["completed"] != false + || receipt["outcome"] != "held" + || receipt["request_sha256"].as_str().is_none_or(|value| value.len() != 64) + || receipt["request_headers_sha256"] + .as_str() + .is_none_or(|value| value.len() != 64) + || (receipt["auth_nonce"].as_str().is_none_or(str::is_empty) + && receipt["replay_nonce"].as_str().is_none_or(str::is_empty)) + || (receipt["signature_sha256"].as_str().is_none_or(|value| value.len() != 64) + && receipt["replay_signature_sha256"] + .as_str() + .is_none_or(|value| value.len() != 64)) + { + return Err(format!("proxy did not retain the original signed production ACK unsent: {receipt}").into()); + } + Ok(()) +} + +fn assert_forwarded_ack(receipt: &Value, ack: &Value, cleared: u64) -> TestResult { + if receipt["outcome"] != "forwarded" + || receipt["forwarded"] != true + || receipt["backend_complete"] != true + || receipt["response"]["status"] != 200 + || receipt["response"]["grpc_status"] != "0" + || receipt["response"]["ack"]["owner_id"] != ack["owner_id"] + || receipt["response"]["ack"]["instance_id"] != ack["instance_id"] + || receipt["response"]["ack"]["supported"] != true + || receipt["response"]["ack"]["cleared"].as_u64() != Some(cleared) + || receipt["response"]["ack"]["response_proof_bytes"] + .as_u64() + .is_none_or(|count| count == 0) + || receipt["response"]["sha256"].as_str().is_none_or(|value| value.len() != 64) + || receipt["response"]["headers_sha256"] + .as_str() + .is_none_or(|value| value.len() != 64) + { + return Err(format!("retained ACK lacks an authenticated backend success clearing {cleared}: {receipt}").into()); + } + Ok(()) +} + +fn dirty_generations(observation: &Value) -> TestResult> { + let snapshot = &observation["detail"]["dirty_snapshot"]; + let entries = snapshot["buckets"] + .as_array() + .ok_or("receiver omitted remaining dirty buckets")?; + if snapshot["complete"] != true || snapshot["pending_bucket_count"].as_u64() != Some(u64::try_from(entries.len())?) { + return Err(format!("receiver dirty snapshot is incomplete or inconsistent: {snapshot}").into()); + } + let mut result = BTreeMap::new(); + for entry in entries { + let bucket = entry["bucket"].as_str().ok_or("receiver dirty entry omitted bucket")?; + let generation = entry["generation"] + .as_u64() + .filter(|value| *value > 0) + .ok_or("receiver omitted dirty generation")?; + if result.insert(bucket.to_string(), generation).is_some() { + return Err("receiver repeated a dirty bucket".into()); + } + } + Ok(result) +} + +async fn applied_ack_after( + run: &PeerBucketRun, + pid: u32, + ack: &Value, + after_sequence: Option, +) -> TestResult<(Value, Value)> { + wait_until( + CYCLE_TIMEOUT, + || async { Ok(receiver_observation_after(run, "ack_after_apply", pid, ack, after_sequence)?.is_some()) }, + "actual retained scoped ACK compare-and-clear", + ) + .await?; + let before = receiver_observation_after(run, "ack_before_apply", pid, ack, after_sequence)? + .ok_or("retained ACK omitted its receiver pre-apply observation")?; + let after = receiver_observation_after(run, "ack_after_apply", pid, ack, after_sequence)? + .ok_or("retained ACK omitted its receiver post-apply observation")?; + if sequence(&before)? >= sequence(&after)? { + return Err("retained ACK receiver observations are out of order".into()); + } + Ok((before, after)) +} + +fn ack_deadline_log(path: &str, offset: u64) -> TestResult> { + use std::io::{BufRead, Seek}; + + let mut file = std::fs::File::open(path)?; + file.seek(std::io::SeekFrom::Start(offset))?; + for line in std::io::BufReader::new(file).lines() { + let line = line?; + if line.contains("scoped dirty usage acknowledgement deadline elapsed") + && line.contains("remote_dirty_usage_acknowledgement_pending") + { + return Ok(Some(line)); + } + } + Ok(None) +} + +fn converged_shards(run: &PeerBucketRun) -> TestResult> { + let mut physical = Vec::new(); + for object in &run.objects { + for (node_index, node) in run.dist.cluster.nodes.iter().enumerate() { + let mut complete = Vec::new(); + for drive in &node.data_dirs { + let census = crate::chaos::census_object_version_on_disk( + std::path::Path::new(drive), + &object.bucket, + &object.key, + Some(&object.version_id), + )?; + if census.has_xl_meta { + if !census.is_complete() || (census.data_blocks, census.parity_blocks) != (Some(2), Some(2)) { + return Err(format!("converged version lost an acknowledged EC:2 shard on {drive}: {census:?}").into()); + } + complete.push(json!({"drive": drive, "census": census})); + } + } + if complete.len() != 1 { + return Err(format!("converged version must retain one complete shard on node {node_index}: {object:?}").into()); + } + physical.push(json!({"object": object, "node": node_index, "shards": complete})); + } + } + Ok(physical) +} + +// The caller retains each real ACK only after this exact hot-window candidate +// has completed every set and published its verified root. +async fn queued_hot_cycle(run: &mut PeerBucketRun, leader: usize, producer: usize, start_gate: &Value) -> TestResult { + queued_hot_cycle_until(run, leader, producer, start_gate, "root_durable").await +} + +async fn queued_hot_cycle_until( + run: &mut PeerBucketRun, + leader: usize, + producer: usize, + start_gate: &Value, + root_stage: &str, +) -> TestResult { + let cycle = identity(start_gate)?; + let leader_pid = run.pid(leader)?; + let [cold_a, cold_b, hot] = run.buckets.clone(); + let all = run.buckets.iter().cloned().collect::>(); + let cold = BTreeSet::from([cold_a, cold_b]); + let address = run.peer_address(producer); + run.put(producer, &hot, &format!("queued-start-{}", cycle.1), 157).await?; + run.barrier.arm("before_post_walk_snapshot", leader_pid, Some(cycle))?; + run.barrier.release("before_start_snapshot", leader_pid)?; + let start = run + .barrier + .wait_stage("start_snapshot", leader_pid, Some(cycle), false) + .await?; + let start_peer = peer_for_address(&start, &address)?; + assert_snapshot_window(&start, start_peer)?; + let start_tuples = tuples(&start_peer["buckets"])?; + if start_tuples.keys().cloned().collect::>() != all { + return Err("held ACK did not leave all three producer buckets dirty for the next actual scan".into()); + } + run.barrier + .wait_stage("before_post_walk_snapshot", leader_pid, Some(cycle), true) + .await?; + run.put(producer, &hot, &format!("queued-post-{}", cycle.1), 197).await?; + run.barrier.arm(root_stage, leader_pid, Some(cycle))?; + run.barrier.release("before_post_walk_snapshot", leader_pid)?; + let post = run + .barrier + .wait_stage("post_walk_snapshot", leader_pid, Some(cycle), false) + .await?; + let post_peer = peer_for_address(&post, &address)?; + assert_snapshot_window(&post, post_peer)?; + let post_tuples = tuples(&post_peer["buckets"])?; + let identities = peer_identities(&start["detail"]["peers"])?; + if peer_identities(&post["detail"]["peers"])? != identities + || cold.iter().any(|bucket| start_tuples.get(bucket) != post_tuples.get(bucket)) + || post_tuples + .get(&hot) + .zip(start_tuples.get(&hot)) + .is_none_or(|(after, before)| after.0 != before.0 || after.1 <= before.1) + { + return Err("queued ACK cycle did not preserve cold tuples while real hot writes advanced".into()); + } + let candidate = run + .barrier + .wait_stage("cycle_candidate", leader_pid, Some(cycle), false) + .await?; + assert_scope(&candidate, &all, &start_tuples)?; + if candidate["detail"]["peer_certification"] != true + || peer_identities(&candidate["detail"]["certified_peers"])? != identities + || names(&candidate["detail"]["pending_buckets"])? != BTreeSet::from([hot]) + || candidate["detail"]["converged"] != false + { + return Err(format!("queued ACK candidate did not certify only cold with hot pending: {candidate}").into()); + } + let root = run.barrier.wait_stage(root_stage, leader_pid, Some(cycle), true).await?; + if root_stage == "root_durable" { + assert_root_binding(&candidate, &root)?; + } else if root_stage != "root_before_publish" + || root["detail"]["candidate_sha256"] != candidate["detail"]["candidate_sha256"] + || root["detail"]["coverage_sha256"] != candidate["detail"]["coverage_sha256"] + { + return Err("pre-publication barrier does not identify the sealed candidate".into()); + } + Ok(json!({"start_gate": start_gate, "start": start, "post": post, "candidate": candidate, "root": root})) +} + +fn complete_dirty_tuples(snapshot: &Value) -> TestResult> { + let entries = tuples(&snapshot["buckets"])?; + if snapshot["complete"] != true + || snapshot["pending_bucket_count"].as_u64() != Some(u64::try_from(entries.len())?) + || snapshot["instance_id"].as_str().is_none_or(str::is_empty) + || snapshot["owner_id"].as_str().is_none_or(str::is_empty) + { + return Err(format!("crash window requires a complete authenticated dirty snapshot: {snapshot}").into()); + } + Ok(entries) +} + +fn assert_physical_root(root: &Value, physical: &Value) -> TestResult { + let usage: DataUsageInfo = serde_json::from_value(physical["usage"].clone())?; + if physical["root_version"] != root["detail"]["root_version"] + || physical["root_sha256"] != root["detail"]["root_sha256"] + || (usage.scanner_epoch.zip(usage.scanner_cycle)) != Some(identity(root)?) + { + return Err("physical authoritative root differs from the actual committed version, digest or cycle".into()); + } + Ok(()) +} + +fn persisted_usage_fields(usage: &Value) -> TestResult { + let mut fields = usage.as_object().ok_or("usage comparison requires an object")?.clone(); + // Admin fills these from live storage statistics; all other fields must + // still match the persisted scanner snapshot. Leave both raw receipts intact. + for field in ["total_capacity", "total_free_capacity", "total_used_capacity"] { + fields.remove(field); + } + Ok(Value::Object(fields)) +} + +fn assert_walked_buckets(run: &PeerBucketRun, cycle: (u64, u64), expected: &BTreeSet) -> TestResult { + let actual = run + .barrier + .observations(false)? + .iter() + .filter(|event| event["stage"] == "bucket_walk" && identity(event).ok() == Some(cycle)) + .map(|event| { + event["detail"]["bucket"] + .as_str() + .map(str::to_string) + .ok_or_else(|| "crash recovery bucket walk omitted its bucket".into()) + }) + .collect::>>()?; + if actual != *expected { + return Err(format!("crash recovery physically walked {actual:?}, expected {expected:?}").into()); + } + Ok(()) +} + +async fn crash_window(stage: &str) -> TestResult { + init_logging(); + let receiver_crash = matches!(stage, "ack_before_apply" | "ack_after_apply"); + if !receiver_crash && !matches!(stage, "root_before_publish" | "root_durable") { + return Err("unknown scanner crash window".into()); + } + let mut run = PeerBucketRun::start_for_crash(stage).await?; + let executable = run.executable_identity()?; + let scheduler_config = run.assert_default_cadence().await?; + let baseline = run.quiet_baseline().await?; + let leader = run.leader().await?; + let leader_pid = run.pid(leader)?; + let producer = (leader + 1) % 4; + let producer_pid = run.pid(producer)?; + let target = if receiver_crash { producer } else { leader }; + let old_process = run.dirty_snapshot(target).await?; + let [cold_a, cold_b, hot] = run.buckets.clone(); + let cold = BTreeSet::from([cold_a.clone(), cold_b.clone()]); + let all = run.buckets.iter().cloned().collect::>(); + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + run.barrier.start_session(&run.buckets)?; + run.put(leader, &hot, "wake-crash-window", 149).await?; + let start_gate = run + .barrier + .wait_stage("before_start_snapshot", leader_pid, None, true) + .await?; + let cycle = identity(&start_gate)?; + if baseline.scanner_epoch != Some(cycle.0) || baseline.scanner_cycle.is_none_or(|previous| previous >= cycle.1) { + return Err("crash window did not start under the observed baseline coordinator".into()); + } + let old_root = run.authoritative_root()?; + if persisted_usage_fields(&old_root["usage"])? != persisted_usage_fields(&serde_json::to_value(&baseline)?)? { + return Err("physical pre-crash root does not match the authoritative baseline".into()); + } + for bucket in &run.buckets.clone() { + run.put(producer, bucket, "crash-start-window", 137).await?; + } + let root_stage = if stage == "root_before_publish" { + stage + } else { + "root_durable" + }; + let window = queued_hot_cycle_until(&mut run, leader, producer, &start_gate, root_stage).await?; + let candidate = &window["candidate"]; + let root = &window["root"]; + let start_peer = peer_for_address(&window["start"], &run.peer_address(producer))?; + let post_tuples = tuples(&peer_for_address(&window["post"], &run.peer_address(producer))?["buckets"])?; + let before_dirty = run.dirty_snapshot(producer).await?; + if complete_dirty_tuples(&before_dirty)? != post_tuples + || before_dirty["owner_id"] != start_peer["owner_id"] + || before_dirty["instance_id"] != start_peer["instance_id"] + || run.leader().await? != leader + { + return Err("actual producer state or coordinator changed before the crash window".into()); + } + let planned_ack = candidate["detail"]["acks"] + .as_array() + .ok_or("candidate omitted ACK plan")? + .iter() + .find(|ack| { + ack["kind"] == "scoped" + && ack["owner_id"] == start_peer["owner_id"] + && ack["instance_id"] == start_peer["instance_id"] + }) + .ok_or("candidate omitted the actual producer's scoped ACK")? + .clone(); + let expected_cold = post_tuples + .iter() + .filter(|(bucket, _)| cold.contains(*bucket)) + .map(|(bucket, tuple)| (bucket.clone(), tuple.clone())) + .collect::>(); + if tuples(&planned_ack["entries"])? != expected_cold || sequence(candidate)? >= sequence(root)? { + return Err("crash window did not seal the exact cold tuples before root publication".into()); + } + let root_at_gate = run.authoritative_root()?; + if stage == "root_before_publish" { + if root_at_gate != old_root { + return Err("uncommitted candidate already replaced the authoritative root".into()); + } + } else { + assert_physical_root(root, &root_at_gate)?; + let usage: DataUsageInfo = serde_json::from_value(root_at_gate["usage"].clone())?; + if usage.usage_snapshot_converged != Some(false) + || usage.usage_snapshot_pending_buckets != vec![hot.clone()] + || !run.usage_matches(&usage, &[&cold_a, &cold_b]) + { + return Err("committed crash-window root lost cold usage or the hot pending obligation".into()); + } + } + let before_shards = converged_shards(&run)?; + // Start before either queue retention or the production sender can begin; + // forwarding cannot reset the age of a previously held request. + let ack_started_at = std::time::Instant::now(); + let queue = run.scoped_ack_queue(producer, start_peer, cold, 1)?; + let mut ack_send = Value::Null; + let mut retained = Value::Null; + let mut before_apply = Value::Null; + let mut after_apply = Value::Null; + let mut crash; + if receiver_crash { + run.barrier.arm_receiver(stage, producer_pid, start_peer)?; + run.barrier.release("root_durable", leader_pid)?; + ack_send = run.barrier.wait_stage("ack_send", leader_pid, Some(cycle), false).await?; + let ack = scoped_ack_for_peer(root, &ack_send, start_peer)?; + if tuples(&ack["entries"])? != expected_cold { + return Err("production sender changed the sealed cold ACK tuples".into()); + } + queue.wait_for_packets(1).await?; + retained = queue.receipts().into_iter().next().ok_or("crash ACK packet disappeared")?; + assert_retained_ack(&retained, ack, &run.dist.cluster.nodes[producer].address)?; + if ack_started_at.elapsed() >= std::time::Duration::from_secs(5) { + return Err("receiver crash window expired while the original ACK was held".into()); + } + let forwarded_at = std::time::Instant::now(); + queue.settle(0, ScopedAckFault::Forward)?; + let receiver_gate = run.barrier.wait_stage(stage, producer_pid, None, true).await?; + if receiver_observation(&run, stage, producer_pid, ack)?.as_ref() != Some(&receiver_gate) { + return Err("armed receiver barrier does not identify the original signed ACK".into()); + } + if ack_started_at.elapsed() >= std::time::Duration::from_secs(5) { + return Err("receiver crash window expired before SIGKILL".into()); + } + // Kill while the unchanged five-second receiver operation is still + // paused. A timed-out hook is not the requested crash window. + crash = run.kill_at(producer, &receiver_gate)?; + let kill_elapsed = forwarded_at.elapsed(); + let total_kill_elapsed = ack_started_at.elapsed(); + if kill_elapsed >= std::time::Duration::from_secs(5) || total_kill_elapsed >= std::time::Duration::from_secs(5) { + return Err("receiver kill exceeded the real scoped ACK operation deadline".into()); + } + crash["forward_to_sigkill_ms"] = json!(u64::try_from(kill_elapsed.as_millis())?); + crash["total_to_sigkill_ms"] = json!(u64::try_from(total_kill_elapsed.as_millis())?); + before_apply = receiver_observation(&run, "ack_before_apply", producer_pid, ack)? + .ok_or("crashed receiver omitted its pre-apply state")?; + let expected_before = post_tuples + .iter() + .map(|(bucket, (_, generation))| (bucket.clone(), *generation)) + .collect::>(); + if dirty_generations(&before_apply)? != expected_before { + return Err("receiver did not retain the exact original dirty map before apply".into()); + } + let observed_after = receiver_observation(&run, "ack_after_apply", producer_pid, ack)?; + if stage == "ack_before_apply" { + if observed_after.is_some() { + return Err("ACK applied despite killing the receiver at its pre-apply barrier".into()); + } + } else { + after_apply = observed_after.ok_or("receiver was killed without observed compare-and-remove")?; + if sequence(&before_apply)? >= sequence(&after_apply)? + || after_apply["detail"]["cleared"] != 2 + || dirty_generations(&after_apply)? != BTreeMap::from([(hot.clone(), post_tuples[&hot].1)]) + { + return Err("post-apply crash did not clear exactly cold while conserving the hot generation".into()); + } + } + } else { + crash = run.kill_at(leader, root)?; + if run.barrier.observations(false)?.iter().any(|event| { + event["pid"].as_u64() == Some(u64::from(leader_pid)) + && identity(event).ok() == Some(cycle) + && (event["stage"] == "ack_send" || (stage == "root_before_publish" && event["stage"] == "root_durable")) + }) || receiver_observation(&run, "ack_before_apply", producer_pid, &planned_ack)?.is_some() + || receiver_observation(&run, "ack_after_apply", producer_pid, &planned_ack)?.is_some() + || !queue.receipts().is_empty() + { + return Err("killed coordinator published or acknowledged past its held crash boundary".into()); + } + } + // Read before any restart or recovery work. No root version is selected + // from disagreeing replicas, and an unexpected successor publication fails + // the controlled window instead of being mistaken for preservation. + let root_after_kill = run.authoritative_root()?; + if root_after_kill != root_at_gate { + return Err("authoritative root changed outside the controlled crash boundary".into()); + } + let transport = if receiver_crash { + let receipt = queue.wait_for_completion(0).await?; + let total_transport_elapsed = ack_started_at.elapsed(); + let transport_elapsed = receipt["completed_at_ms"] + .as_u64() + .zip(receipt["forwarded_at_ms"].as_u64()) + .and_then(|(completed, forwarded)| completed.checked_sub(forwarded)); + if receipt["completed"] != true + || receipt["forwarded"] != true + || receipt["backend_complete"] != false + || receipt["outcome"] != "transport_failed_or_expired" + || !receipt["response"].is_null() + || receipt["request_frame_hex"] != retained["request_frame_hex"] + || receipt["request_sha256"] != retained["request_sha256"] + || receipt["request_headers_sha256"] != retained["request_headers_sha256"] + || receipt["auth_nonce"] != retained["auth_nonce"] + || receipt["replay_nonce"] != retained["replay_nonce"] + || transport_elapsed.is_none_or(|millis| millis >= 5_000) + || total_transport_elapsed >= std::time::Duration::from_secs(5) + { + return Err("receiver crash did not lose the original signed ACK response in transport".into()); + } + crash["total_to_transport_completion_ms"] = json!(u64::try_from(total_transport_elapsed.as_millis())?); + receipt + } else { + let after = run.dirty_snapshot(producer).await?; + if after != before_dirty { + return Err("coordinator crash changed the surviving producer's exact dirty responsibility".into()); + } + after + }; + drop(queue); + run.write_evidence( + "crash-boundary", + &json!({"stage": stage, "executable": executable, "scheduler_config": scheduler_config, + "baseline": baseline, "topology": run.topology, "old_process": old_process, "old_root": old_root, + "window": window, "before_dirty": before_dirty, "planned_ack": planned_ack, "ack_send": ack_send, + "retained_packet": retained, "before_apply": before_apply, "after_apply": after_apply, "transport": transport, + "root_at_gate": root_at_gate, "root_after_kill": root_after_kill, "crash": crash, + "objects": run.objects, "before_shards": before_shards}), + )?; + run.barrier.disarm()?; + let before_restart = run.barrier.observations(false)?; + let restarted = run.restart_after_crash(&crash, &old_process).await?; + run.write_evidence( + "restarted-process", + &json!({"crash": crash, "previous": old_process, "restarted": restarted, + "drives": run.drive_identity(target)?, "before_restart_observations": before_restart}), + )?; + + // No S3 mutation is allowed between the crash and this convergence check. + // An empty volatile map in the new process cannot account for old versions. + let recovered = run.quiet_baseline().await?; + let recovery_cycle = recovered + .scanner_epoch + .zip(recovered.scanner_cycle) + .ok_or("recovery root omitted cycle")?; + if recovery_cycle == cycle || recovered == baseline { + return Err("restart did not produce a fresh authoritative recovery root".into()); + } + let recovery_root = run.authoritative_root()?; + let observations = run.barrier.observations(false)?; + run.write_evidence( + "automatic-crash-recovery-observed", + &json!({"restarted": restarted, "usage": recovered, "root": recovery_root, + "objects": run.objects, "observations": observations}), + )?; + if persisted_usage_fields(&recovery_root["usage"])? != persisted_usage_fields(&serde_json::to_value(&recovered)?)? { + return Err("automatic recovery admin state differs from the physical authoritative root".into()); + } + let recovery_durable = observations + .iter() + .find(|event| { + event["stage"] == "root_durable" + && identity(event).ok() == Some(recovery_cycle) + && event["detail"]["root_version"] == recovery_root["root_version"] + && event["detail"]["root_sha256"] == recovery_root["root_sha256"] + }) + .ok_or("automatic recovery lacks an actual durable root receipt")?; + let recovery_candidate = observations + .iter() + .find(|event| { + event["stage"] == "cycle_candidate" + && event["pid"] == recovery_durable["pid"] + && identity(event).ok() == Some(recovery_cycle) + }) + .ok_or("automatic recovery lacks actual per-set completion")?; + let recovery_sequence = sequence(recovery_candidate)?; + if before_restart.iter().any(|event| { + event["pid"] == recovery_candidate["pid"] + && event["sequence"] + .as_u64() + .is_some_and(|sequence| sequence >= recovery_sequence) + }) { + return Err("automatic recovery reused a candidate observed before the replacement process started".into()); + } + assert_completed_scope(recovery_candidate, &all, &post_tuples, true)?; + assert_root_binding(recovery_candidate, recovery_durable)?; + assert_physical_root(recovery_durable, &recovery_root)?; + assert_walked_buckets(&run, recovery_cycle, &all)?; + if sequence(recovery_candidate)? >= sequence(recovery_durable)? + || recovery_candidate["detail"]["converged"] != true + || !names(&recovery_candidate["detail"]["pending_buckets"])?.is_empty() + { + return Err("automatic restart recovery did not conservatively complete all bucket obligations".into()); + } + run.verify_objects().await?; + let recovered_shards = converged_shards(&run)?; + if recovered_shards != before_shards { + return Err("restart changed an acknowledged physical version shard".into()); + } + run.write_evidence( + "automatic-crash-recovery", + &json!({"restarted": restarted, "usage": recovered, + "root": recovery_root, "candidate": recovery_candidate, "durable": recovery_durable, + "objects": run.objects, "physical": recovered_shards, "observations": observations}), + )?; + + let new_leader = run.leader().await?; + // Recovery may elect another coordinator. Keep the ACK receiver remote, + // and bind the replacement through the role it actually holds now. + let fresh_receiver = if target == new_leader { (new_leader + 1) % 4 } else { target }; + run.write_evidence( + "fresh-process-roles", + &json!({"leader_before_crash": leader, "leader_after_recovery": new_leader, + "coordinator_pid": run.pid(new_leader)?, "receiver": fresh_receiver, + "receiver_pid": run.pid(fresh_receiver)?, "replacement": target, + "replacement_pid": restarted["pid"], + "replacement_role": if target == new_leader { "coordinator" } else { "receiver" }}), + )?; + let fresh_ack = fresh_ack_after_restart(&mut run, new_leader, fresh_receiver, &post_tuples).await?; + if (fresh_receiver == target && fresh_ack["peer"]["instance_id"] != restarted["instance_id"]) + || (new_leader == target && fresh_ack["coordinator_instance"] != restarted["instance_id"]) + { + return Err("fresh scoped ACK did not bind the actual replacement process instance".into()); + } + run.barrier.stop_session()?; + run.verify_objects().await?; + let physical = converged_shards(&run)?; + if run.executable_identity()? != executable { + return Err("crash recovery restarted a different executable".into()); + } + run.write_evidence( + "crash-result", + &json!({"stage": stage, "crash": crash, "restarted": restarted, + "automatic_recovery_root": recovery_root, "fresh_ack": fresh_ack, "executable": executable, + "objects": run.objects, "physical": physical, "observations": run.barrier.observations(false)?}), + )?; + Ok(()) +} + +async fn fresh_ack_after_restart( + run: &mut PeerBucketRun, + leader: usize, + receiver: usize, + incarnations: &BTreeMap, +) -> TestResult { + let leader_pid = run.pid(leader)?; + let receiver_pid = run.pid(receiver)?; + let hot = run.buckets[2].clone(); + let expected = BTreeSet::from([hot.clone()]); + let previous = run.dirty_snapshot(receiver).await?; + if !complete_dirty_tuples(&previous)?.is_empty() { + return Err("automatic recovery left receiver dirty state before fresh-write proof".into()); + } + let previous_sequence = run + .barrier + .observations(false)? + .iter() + .filter(|event| event["pid"].as_u64() == Some(u64::from(leader_pid))) + .map(sequence) + .collect::>>()? + .into_iter() + .max(); + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + run.put(leader, &hot, "wake-fresh-process-proof", 151).await?; + let gate = match previous_sequence { + Some(previous) => { + run.barrier + .wait_stage_after("before_start_snapshot", leader_pid, None, true, previous) + .await? + } + None => { + run.barrier + .wait_stage("before_start_snapshot", leader_pid, None, true) + .await? + } + }; + let cycle = identity(&gate)?; + let write = run.put(receiver, &hot, "fresh-process-proof", 167).await?; + run.barrier.arm("root_durable", leader_pid, Some(cycle))?; + run.barrier.release("before_start_snapshot", leader_pid)?; + let start = run + .barrier + .wait_stage("start_snapshot", leader_pid, Some(cycle), false) + .await?; + let peer = peer_for_address(&start, &run.peer_address(receiver))?; + assert_snapshot_window(&start, peer)?; + let entries = complete_dirty_tuples(peer)?; + if peer["owner_id"] != previous["owner_id"] + || peer["instance_id"] != previous["instance_id"] + || entries.keys().cloned().collect::>() != expected + || entries[&hot].0 != incarnations[&hot].0 + { + return Err("fresh write did not establish new evidence under the surviving bucket incarnation and process".into()); + } + let candidate = run + .barrier + .wait_stage("cycle_candidate", leader_pid, Some(cycle), false) + .await?; + assert_scope(&candidate, &expected, &entries)?; + if candidate["detail"]["converged"] != true || !names(&candidate["detail"]["pending_buckets"])?.is_empty() { + return Err("fresh-process candidate did not converge its new write".into()); + } + let root = run.barrier.wait_stage("root_durable", leader_pid, Some(cycle), true).await?; + assert_root_binding(&candidate, &root)?; + assert_walked_buckets(run, cycle, &expected)?; + let physical_root = run.authoritative_root()?; + assert_physical_root(&root, &physical_root)?; + let usage: DataUsageInfo = serde_json::from_value(physical_root["usage"].clone())?; + if !run.usage_matches(&usage, &run.buckets.iter().map(String::as_str).collect::>()) { + return Err("fresh process publication lost an acknowledged object/version total".into()); + } + let queue = run.scoped_ack_queue(receiver, peer, expected, 1)?; + run.barrier.disarm()?; + run.barrier.release("root_durable", leader_pid)?; + let send = run.barrier.wait_stage("ack_send", leader_pid, Some(cycle), false).await?; + let ack = scoped_ack_for_peer(&root, &send, peer)?; + if tuples(&ack["entries"])? != entries { + return Err("fresh process ACK did not carry its own newly certified dirty tuple".into()); + } + queue.wait_for_packets(1).await?; + let retained = queue + .receipts() + .into_iter() + .next() + .ok_or("fresh process ACK packet disappeared")?; + assert_retained_ack(&retained, ack, &run.dist.cluster.nodes[receiver].address)?; + queue.settle(0, ScopedAckFault::Forward)?; + let receipt = queue.wait_for_completion(0).await?; + assert_forwarded_ack(&receipt, ack, 1)?; + let (before, after) = applied_ack_after(run, receiver_pid, ack, None).await?; + if dirty_generations(&before)? != BTreeMap::from([(hot, entries[&run.buckets[2]].1)]) + || !dirty_generations(&after)?.is_empty() + || !complete_dirty_tuples(&run.dirty_snapshot(receiver).await?)?.is_empty() + { + return Err("fresh process clearing lacked the matching pre-apply tuple and actual successful removal".into()); + } + Ok(json!({"write": write, "peer": peer, "start": start, + "coordinator_instance": start["detail"]["activity_before"][""]["instance_id"], "candidate": candidate, "root": root, + "physical_root": physical_root, "send": send, "retained": retained, "transport": receipt, + "before_apply": before, "after_apply": after})) +} + +async fn reordered_scoped_acks(newer_generation: bool) -> TestResult { + init_logging(); + let label = if newer_generation { + "ack-generation-reorder" + } else { + "ack-logical-duplicate" + }; + let mut run = PeerBucketRun::start_with_ack_faults(label).await?; + let scheduler_config = run.assert_default_cadence().await?; + let baseline = run.quiet_baseline().await?; + let leader = run.leader().await?; + let leader_pid = run.pid(leader)?; + let producer = (leader + 1) % 4; + let producer_pid = run.pid(producer)?; + let address = run.peer_address(producer); + let [cold_a, cold_b, hot] = run.buckets.clone(); + let cold = BTreeSet::from([cold_a.clone(), cold_b.clone()]); + let all = run.buckets.iter().cloned().collect::>(); + let log_path = run.dist.cluster.node_capture_log_paths[leader] + .clone() + .ok_or("coordinator has no captured log")?; + run.write_evidence( + "baseline", + &json!({"topology": run.topology, "baseline": baseline, + "scheduler_config": scheduler_config, "objects": run.objects, "leader_node": leader, "producer_node": producer}), + )?; + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + run.barrier.start_session(&run.buckets)?; + run.put(leader, &hot, "queued-wake-start", 149).await?; + let mut start_gate = run + .barrier + .wait_stage("before_start_snapshot", leader_pid, None, true) + .await?; + let first_cycle = identity(&start_gate)?; + if baseline.scanner_epoch != Some(first_cycle.0) || baseline.scanner_cycle.is_none_or(|cycle| cycle >= first_cycle.1) { + return Err("retained ACK window did not follow the quiet baseline under the same leader epoch".into()); + } + for bucket in &all { + run.put(producer, bucket, "queued-initial-generation", 137).await?; + } + let mut writer = Some(run.start_hot_writer(producer, &hot)?); + let mut hot_stream = Vec::new(); + let mut queue = None; + let mut cycles = Vec::new(); + let mut deadline_log = None; + for index in 0..2 { + let cycle = identity(&start_gate)?; + if cycle.0 != first_cycle.0 || run.leader().await? != leader || (index == 1 && cycle.1 <= first_cycle.1) { + return Err("retained ACK cycles changed coordinator or epoch".into()); + } + let mut observed = queued_hot_cycle(&mut run, leader, producer, &start_gate).await?; + let peer = peer_for_address(&observed["start"], &address)?.clone(); + if index == 0 { + queue = Some(run.scoped_ack_queue(producer, &peer, cold.clone(), 2)?); + } + let hot_writer = writer.as_mut().ok_or("hot writer ended before the second candidate")?; + hot_writer.wait_for_completed(hot_writer.completed() + 1).await?; + if index == 1 { + hot_stream = writer.take().ok_or("hot writer already joined")?.finish().await?; + run.objects.extend(hot_stream.clone()); + } + run.put(producer, &hot, &format!("queued-after-root-{}", cycle.1), 257) + .await?; + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + run.put(leader, &hot, &format!("queued-wake-next-{}", cycle.1), 151).await?; + let log_offset = std::fs::metadata(&log_path)?.len(); + run.barrier.release("root_durable", leader_pid)?; + let send = run.barrier.wait_stage("ack_send", leader_pid, Some(cycle), false).await?; + let ack = scoped_ack_for_peer(&observed["root"], &send, &peer)?.clone(); + let entries = tuples(&ack["entries"])?; + let snapshot = tuples(&peer["buckets"])?; + if entries.keys().cloned().collect::>() != cold + || entries.iter().any(|(bucket, entry)| snapshot.get(bucket) != Some(entry)) + { + return Err("retained production ACK did not bind exactly the certified cold tuples".into()); + } + let ack_queue = queue.as_ref().ok_or("ACK queue was not armed")?; + ack_queue.wait_for_packets(index + 1).await?; + let captured = ack_queue.receipts()[index].clone(); + assert_retained_ack(&captured, &ack, &run.dist.cluster.nodes[producer].address)?; + observed["send"] = send; + observed["ack"] = ack; + observed["captured"] = captured; + run.write_evidence(&format!("queued-cycle-{index}"), &observed)?; + cycles.push(observed); + if index == 0 { + // No proxy response or request loss is injected. Reaching the next + // cycle requires the genuine caller deadline to end this attempt. + start_gate = run + .barrier + .wait_stage_after("before_start_snapshot", leader_pid, None, true, sequence(&start_gate)?) + .await?; + wait_until( + CYCLE_TIMEOUT, + || async { Ok(ack_deadline_log(&log_path, log_offset)?.is_some()) }, + "actual production ACK deadline", + ) + .await?; + deadline_log = ack_deadline_log(&log_path, log_offset)?; + assert_retained_ack(&ack_queue.receipts()[0], &cycles[0]["ack"], &run.dist.cluster.nodes[producer].address)?; + if receiver_observation(&run, "ack_before_apply", producer_pid, &cycles[0]["ack"])?.is_some() + || receiver_observation(&run, "ack_after_apply", producer_pid, &cycles[0]["ack"])?.is_some() + { + return Err("held first ACK reached the receiver before its caller expired".into()); + } + if newer_generation { + for bucket in &cold { + run.put(producer, bucket, "queued-newer-generation", 211).await?; + } + } + } + } + let queue = queue.ok_or("two production ACKs were not retained")?; + let first_ack = &cycles[0]["ack"]; + let second_ack = &cycles[1]["ack"]; + let first_entries = tuples(&first_ack["entries"])?; + let second_entries = tuples(&second_ack["entries"])?; + if first_ack["owner_id"] != second_ack["owner_id"] + || first_ack["instance_id"] != second_ack["instance_id"] + || cold.iter().any(|bucket| { + let before = &first_entries[bucket]; + let after = &second_entries[bucket]; + before.0 != after.0 + || if newer_generation { + after.1 <= before.1 + } else { + after.1 != before.1 + } + }) + { + return Err("second real cycle did not produce the required same-owner cold generation relation".into()); + } + let first_packet = &cycles[0]["captured"]; + let second_packet = &cycles[1]["captured"]; + for field in ["auth_nonce", "replay_nonce", "signature_sha256", "replay_signature_sha256"] { + if !first_packet[field].is_null() && (second_packet[field].is_null() || first_packet[field] == second_packet[field]) { + return Err(format!("separate production ACKs reused authentication field {field}").into()); + } + } + if first_packet["request_sha256"] == second_packet["request_sha256"] + || first_packet["request_headers_sha256"] == second_packet["request_headers_sha256"] + { + return Err("logical ACK duplication reused a raw signed request instead of a fresh production request".into()); + } + queue.settle(1, ScopedAckFault::Forward)?; + let second_receipt = queue.wait_for_completion(1).await?; + assert_forwarded_ack(&second_receipt, second_ack, 2)?; + let (second_before, second_after) = applied_ack_after(&run, producer_pid, second_ack, None).await?; + let after_second = dirty_generations(&second_after)?; + let second_post = tuples(&peer_for_address(&cycles[1]["post"], &address)?["buckets"])?; + if second_after["detail"]["cleared"] != 2 + || after_second.keys().cloned().collect::>() != BTreeSet::from([hot.clone()]) + || after_second + .get(&hot) + .is_none_or(|generation| *generation <= second_post[&hot].1) + { + return Err("newer delivery did not clear exactly cold while retaining the later hot generation".into()); + } + let recovery_gate = run + .barrier + .wait_stage_after("before_start_snapshot", leader_pid, None, true, sequence(&start_gate)?) + .await?; + let recovery_cycle = identity(&recovery_gate)?; + if recovery_cycle.0 != first_cycle.0 || recovery_cycle.1 <= identity(&cycles[1]["root"])?.1 || run.leader().await? != leader { + return Err("recovery cycle changed coordinator, epoch, or order".into()); + } + if newer_generation { + for bucket in &cold { + run.put(producer, bucket, "queued-newest-generation", 307).await?; + } + } + assert_retained_ack(&queue.receipts()[0], first_ack, &run.dist.cluster.nodes[producer].address)?; + queue.settle(0, ScopedAckFault::Forward)?; + let first_receipt = queue.wait_for_completion(0).await?; + assert_forwarded_ack(&first_receipt, first_ack, 0)?; + let (first_before, first_after) = applied_ack_after(&run, producer_pid, first_ack, Some(sequence(&second_after)?)).await?; + let remaining = dirty_generations(&first_after)?; + let recovery_scope = if newer_generation { + all.clone() + } else { + BTreeSet::from([hot.clone()]) + }; + if first_after["detail"]["cleared"] != 0 + || first_before["detail"]["dirty_snapshot"] != first_after["detail"]["dirty_snapshot"] + || remaining.keys().cloned().collect::>() != recovery_scope + || remaining.get(&hot) != after_second.get(&hot) + || (newer_generation && cold.iter().any(|bucket| remaining[bucket] <= second_entries[bucket].1)) + || first_receipt["forwarded_at_ms"] + .as_u64() + .zip(second_receipt["backend_completed_at_ms"].as_u64()) + .is_none_or(|(first, second)| first < second) + { + return Err(format!("late old ACK changed responsibility or lost a newer generation: {first_after}").into()); + } + run.barrier.arm("root_durable", leader_pid, Some(recovery_cycle))?; + run.barrier.release("before_start_snapshot", leader_pid)?; + let recovery_start = run + .barrier + .wait_stage("start_snapshot", leader_pid, Some(recovery_cycle), false) + .await?; + let recovery_peer = peer_for_address(&recovery_start, &address)?; + let recovery_entries = tuples(&recovery_peer["buckets"])?; + if recovery_entries + .iter() + .map(|(bucket, entry)| (bucket.clone(), entry.1)) + .collect::>() + != remaining + { + return Err("next real snapshot did not retain the exact generations surviving the old ACK".into()); + } + let recovery_candidate = run + .barrier + .wait_stage("cycle_candidate", leader_pid, Some(recovery_cycle), false) + .await?; + assert_scope(&recovery_candidate, &recovery_scope, &recovery_entries)?; + if !names(&recovery_candidate["detail"]["pending_buckets"])?.is_empty() || recovery_candidate["detail"]["converged"] != true { + return Err("healthy recovery failed to certify the remaining actual generations".into()); + } + let recovery_root = run + .barrier + .wait_stage("root_durable", leader_pid, Some(recovery_cycle), true) + .await?; + assert_root_binding(&recovery_candidate, &recovery_root)?; + let queue_receipts = queue.receipts(); + drop(queue); + run.barrier.disarm()?; + run.barrier.release("root_durable", leader_pid)?; + let recovery_send = run + .barrier + .wait_stage("ack_send", leader_pid, Some(recovery_cycle), false) + .await?; + let recovery_ack = scoped_ack_for_peer(&recovery_root, &recovery_send, recovery_peer)?; + if tuples(&recovery_ack["entries"])? != recovery_entries { + return Err("recovery ACK did not preserve the surviving next-snapshot tuples".into()); + } + let (recovery_before, recovery_after) = + applied_ack_after(&run, producer_pid, recovery_ack, Some(sequence(&first_after)?)).await?; + if recovery_after["detail"]["cleared"].as_u64() != Some(u64::try_from(recovery_scope.len())?) + || !dirty_generations(&recovery_after)?.is_empty() + { + return Err("healthy production ACK did not clear precisely the final retained responsibility".into()); + } + let recovered_usage = run.quiet_baseline().await?; + run.barrier.stop_session()?; + run.verify_objects().await?; + let physical_after_convergence = converged_shards(&run)?; + run.write_evidence( + label, + &json!({"topology": run.topology, "objects": run.objects, "baseline": baseline, + "scheduler_config": scheduler_config, "leader_node": leader, "leader_pid": leader_pid, + "producer_node": producer, "producer_pid": producer_pid, "newer_generation": newer_generation, + "hot_stream": hot_stream, "deadline_log": deadline_log, "cycles": cycles, "queue_receipts": queue_receipts, + "second_before": second_before, "second_after": second_after, "first_before": first_before, "first_after": first_after, + "recovery_scope": recovery_scope, "recovery_gate": recovery_gate, "recovery_start": recovery_start, + "recovery_candidate": recovery_candidate, "recovery_root": recovery_root, "recovery_send": recovery_send, + "recovery_before": recovery_before, "recovery_after": recovery_after, "recovered_usage": recovered_usage, + "physical_after_convergence": physical_after_convergence, + "observations": run.barrier.observations(false)?}), + ) +} + +async fn hot_window(fault: Option) -> TestResult { + init_logging(); + let label = match fault { + None => "hot-window", + Some(ScopedAckFault::RequestLoss) => "ack-request-loss", + Some(ScopedAckFault::ResponseLoss) => "ack-response-loss", + Some(ScopedAckFault::Forward) => return Err("fault scenario needs actual request or response loss".into()), + }; + let mut run = if fault.is_some() { + PeerBucketRun::start_with_ack_faults(label).await? + } else { + PeerBucketRun::start(label).await? + }; + let scheduler_config = run.assert_default_cadence().await?; + let baseline = run.quiet_baseline().await?; + let leader = run.leader().await?; + let leader_pid = run.pid(leader)?; + let producer = (leader + 1) % 4; + let producers = if fault.is_some() { + vec![producer, (leader + 2) % 4] + } else { + vec![producer] + }; + let producer_pid = run.pid(producer)?; + let producer_address = run.peer_address(producer); + let [cold_a, cold_b, hot] = run.buckets.clone(); + let cold = BTreeSet::from([cold_a.clone(), cold_b.clone()]); + let all = run.buckets.iter().cloned().collect::>(); + run.write_evidence( + "baseline", + &json!({"topology": run.topology, "baseline": baseline, "objects": run.objects, "scheduler_config": scheduler_config, + "leader_node": leader, "leader_pid": leader_pid, "producer_node": producer, "producer_pid": producer_pid}), + )?; + + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + run.barrier.start_session(&run.buckets)?; + // A quiet baseline has no pending wake. A local S3 mutation reaches the + // real DirtyUsage path; waiting for the timer would require a full scan. + let start_cycle_wake = run.put(leader, &hot, "wake-start-cycle", 149).await?; + run.write_evidence("start-cycle-wake", &json!({"leader_pid": leader_pid, "put": start_cycle_wake}))?; + let start_gate = run + .barrier + .wait_stage("before_start_snapshot", leader_pid, None, true) + .await?; + let cycle = identity(&start_gate)?; + if baseline.scanner_epoch != Some(cycle.0) || baseline.scanner_cycle.is_none_or(|value| value >= cycle.1) { + return Err("test window did not follow the quiet baseline under the observed leader epoch".into()); + } + // These are actual mutations after leading activity and before the start + // map RPCs. All three buckets must appear on the remote ingress peer. + for node in &producers { + for bucket in [&cold_a, &cold_b, &hot] { + run.put(*node, bucket, "start-window", 137).await?; + } + } + run.barrier.arm("before_post_walk_snapshot", leader_pid, Some(cycle))?; + run.barrier.release("before_start_snapshot", leader_pid)?; + let start = run + .barrier + .wait_stage("start_snapshot", leader_pid, Some(cycle), false) + .await?; + let start_peer = peer_for_address(&start, &producer_address)?; + assert_snapshot_window(&start, start_peer)?; + let start_producers = producers + .iter() + .map(|node| peer_for_address(&start, &run.peer_address(*node)).cloned()) + .collect::>>()?; + for peer in &start_producers { + assert_snapshot_window(&start, peer)?; + } + let start_identities = peer_identities(&start["detail"]["peers"])?; + for node in (0..4).filter(|node| *node != leader) { + peer_for_address(&start, &run.peer_address(node))?; + } + let start_tuples = tuples(&start_peer["buckets"])?; + if start_tuples.keys().cloned().collect::>() != all { + return Err(format!("remote producer did not own all three actual start tuples: {start_peer}").into()); + } + + run.barrier + .wait_stage("before_post_walk_snapshot", leader_pid, Some(cycle), true) + .await?; + let mut hot_in_post_snapshot_window = Vec::new(); + for node in &producers { + hot_in_post_snapshot_window.push(run.put(*node, &hot, "post-walk-window", 193).await?); + } + let mut hot_writer = if producers.len() == 1 { + run.start_hot_writer(producer, &hot)? + } else { + run.start_alternating_hot_writer(&producers, &hot)? + }; + hot_writer.wait_for_completed(producers.len()).await?; + run.barrier.arm("root_durable", leader_pid, Some(cycle))?; + run.barrier.release("before_post_walk_snapshot", leader_pid)?; + let post = run + .barrier + .wait_stage("post_walk_snapshot", leader_pid, Some(cycle), false) + .await?; + let post_peer = peer_for_address(&post, &producer_address)?; + assert_snapshot_window(&post, post_peer)?; + let post_tuples = tuples(&post_peer["buckets"])?; + if peer_identities(&post["detail"]["peers"])? != start_identities { + return Err("a remote peer identity changed inside the certification window".into()); + } + for bucket in &cold { + if start_tuples.get(bucket) != post_tuples.get(bucket) { + return Err(format!("cold bucket {bucket} changed inside the certification window").into()); + } + } + let hot_start = start_tuples.get(&hot).ok_or("hot bucket absent from actual start map")?; + let hot_post = post_tuples.get(&hot).ok_or("hot bucket absent from actual post-walk map")?; + if hot_start.0 != hot_post.0 || hot_post.1 <= hot_start.1 { + return Err("hot S3 operation did not advance the same-incarnation remote dirty tuple".into()); + } + for (node, start_peer) in producers.iter().zip(&start_producers) { + let post_peer = peer_for_address(&post, &run.peer_address(*node))?; + assert_snapshot_window(&post, post_peer)?; + let previous = tuples(&start_peer["buckets"])?; + let current = tuples(&post_peer["buckets"])?; + if previous.keys().cloned().collect::>() != all + || cold.iter().any(|bucket| previous.get(bucket) != current.get(bucket)) + || current + .get(&hot) + .zip(previous.get(&hot)) + .is_none_or(|(after, before)| after.0 != before.0 || after.1 <= before.1) + { + return Err(format!("alternating producer {node} did not preserve cold and advance hot: {post_peer}").into()); + } + } + + let candidate = run + .barrier + .wait_stage("cycle_candidate", leader_pid, Some(cycle), false) + .await?; + assert_scope(&candidate, &all, &start_tuples)?; + if candidate["detail"]["peer_certification"].as_bool() != Some(true) + || peer_identities(&candidate["detail"]["certified_peers"])? != start_identities + || names(&candidate["detail"]["pending_buckets"])? != BTreeSet::from([hot.clone()]) + || candidate["detail"]["converged"].as_bool() != Some(false) + { + return Err(format!("hot-window candidate is not complete with only hot pending: {candidate}").into()); + } + let root = run.barrier.wait_stage("root_durable", leader_pid, Some(cycle), true).await?; + for field in ["candidate_sha256", "coverage_sha256"] { + let digest = candidate["detail"][field].as_str().filter(|value| value.len() == 64); + if digest.is_none() || root["detail"][field].as_str() != digest { + return Err(format!("durable root is not bound to the actual candidate {field}").into()); + } + } + if root["detail"]["root_sha256"].as_str().is_none_or(|value| value.len() != 64) + || root["detail"]["root_version"].is_null() + || !matches!(root["detail"]["outcome"].as_str(), Some("saved" | "already_durable")) + { + return Err(format!("root observation lacks a verified durable publication receipt: {root}").into()); + } + let completed_at_root = hot_writer.completed(); + hot_writer.wait_for_completed(completed_at_root + producers.len()).await?; + let hot_stream = hot_writer.finish().await?; + run.objects.extend(hot_stream.clone()); + // The sealed pending set is immutable. A later write must survive the + // actual receiver's generation check without resealing that candidate. + if hot_stream.iter().map(|receipt| receipt.ingress_node).collect::>() != producers.iter().copied().collect() + || hot_stream + .iter() + .enumerate() + .any(|(index, receipt)| receipt.ingress_node != producers[index % producers.len()]) + { + return Err("acknowledged hot stream did not alternate the actual S3 ingress processes".into()); + } + let mut hot_after_root = Vec::new(); + for node in &producers { + hot_after_root.push(run.put(*node, &hot, "after-root-before-ack", 257).await?); + } + let ack_queue = if fault.is_some() { + Some(run.scoped_ack_queue(producer, start_peer, cold.clone(), 1)?) + } else { + None + }; + run.barrier.arm("before_start_snapshot", leader_pid, None)?; + // Advance the coordinator's local generation after this cycle's start. + // The pending hot bucket then wakes the next scoped cycle immediately. + let next_cycle_wake = run.put(leader, &hot, "wake-next-cycle", 151).await?; + run.write_evidence("next-cycle-wake", &json!({"leader_pid": leader_pid, "put": next_cycle_wake}))?; + run.barrier.release("root_durable", leader_pid)?; + let ack_send = run.barrier.wait_stage("ack_send", leader_pid, Some(cycle), false).await?; + if sequence(&root)? >= sequence(&ack_send)? + || ack_send["detail"]["candidate_sha256"] != root["detail"]["candidate_sha256"] + || ack_send["detail"]["coverage_sha256"] != root["detail"]["coverage_sha256"] + { + return Err("scoped ACK was not ordered after the same candidate's verified root".into()); + } + let expected_ack = ack_send["detail"]["acks"] + .as_array() + .ok_or("sender omitted actual acknowledgements")? + .iter() + .find(|ack| ack["owner_id"] == start_peer["owner_id"] && ack["instance_id"] == start_peer["instance_id"]) + .ok_or("sender did not acknowledge the actual remote producer")?; + let expected_entries = tuples(&expected_ack["entries"])?; + if expected_ack["kind"].as_str() != Some("scoped") + || expected_entries.keys().cloned().collect::>() != cold + || expected_entries + .iter() + .any(|(bucket, entry)| start_tuples.get(bucket) != Some(entry)) + { + return Err(format!("sender did not preserve the certified remote cold tuples: {expected_ack}").into()); + } + let fault_receipt = if let (Some(queue), Some(fault)) = (&ack_queue, fault) { + queue.wait_for_packets(1).await?; + let captured = queue + .receipts() + .into_iter() + .next() + .ok_or("scoped ACK queue omitted its retained packet")?; + run.write_evidence("ack-fault-captured", &captured)?; + if captured["request"]["owner_id"] != expected_ack["owner_id"] + || captured["request"]["instance_id"] != expected_ack["instance_id"] + || tuples(&captured["request"]["entries"])? != expected_entries + || captured["receiver"].as_str() != Some(run.dist.cluster.nodes[producer].address.as_str()) + || captured["forwarded"].as_bool() != Some(false) + || (captured["auth_nonce"].as_str().is_none_or(str::is_empty) + && captured["replay_nonce"].as_str().is_none_or(str::is_empty)) + || (captured["signature_sha256"].as_str().is_none_or(|value| value.len() != 64) + && captured["replay_signature_sha256"] + .as_str() + .is_none_or(|value| value.len() != 64)) + { + return Err(format!("proxy did not retain the original signed cold ACK before forwarding: {captured}").into()); + } + queue.settle(0, fault)?; + let receipt = queue.wait_for_completion(0).await?; + run.write_evidence("ack-fault", &receipt)?; + match fault { + ScopedAckFault::RequestLoss + if receipt["outcome"] == "request_lost" + && receipt["forwarded"] == false + && receipt["backend_complete"] == false => {} + ScopedAckFault::ResponseLoss + if receipt["outcome"] == "response_lost" + && receipt["forwarded"] == true + && receipt["backend_complete"] == true + && receipt["response"]["status"] == 200 + && receipt["response"]["grpc_status"] == "0" + && receipt["response"]["ack"]["owner_id"] == expected_ack["owner_id"] + && receipt["response"]["ack"]["instance_id"] == expected_ack["instance_id"] + && receipt["response"]["ack"]["cleared"] == 2 + && receipt["response"]["ack"]["response_proof_bytes"] + .as_u64() + .is_some_and(|count| count > 0) => {} + _ => return Err(format!("scoped ACK fault lacks its actual transport/backend outcome: {receipt}").into()), + } + receipt + } else { + Value::Null + }; + let find_receiver = |stage: &str| receiver_observation(&run, stage, producer_pid, expected_ack); + let (before_ack, after_ack) = if fault == Some(ScopedAckFault::RequestLoss) { + (Value::Null, Value::Null) + } else { + wait_until( + CYCLE_TIMEOUT, + || async { Ok(find_receiver("ack_after_apply")?.is_some()) }, + "real remote scoped ACK apply", + ) + .await?; + let before_ack = find_receiver("ack_before_apply")?.ok_or("receiver omitted pre-apply observation")?; + let after_ack = find_receiver("ack_after_apply")?.ok_or("receiver omitted post-apply observation")?; + let remaining = &after_ack["detail"]["dirty_snapshot"]; + let remaining_buckets = remaining["buckets"] + .as_array() + .ok_or("receiver omitted actual remaining dirty map")?; + if sequence(&before_ack)? >= sequence(&after_ack)? + || after_ack["detail"]["cleared"].as_u64() != Some(2) + || remaining["complete"].as_bool() != Some(true) + || remaining["pending_bucket_count"].as_u64() != Some(1) + || remaining_buckets.len() != 1 + || remaining_buckets[0]["bucket"].as_str() != Some(hot.as_str()) + || remaining_buckets[0]["generation"] + .as_u64() + .is_none_or(|generation| generation <= hot_post.1) + { + return Err( + format!("receiver did not clear precisely cold while preserving the later hot generation: {after_ack}").into(), + ); + } + (before_ack, after_ack) + }; + + let next_start_gate = run + .barrier + .wait_stage_after("before_start_snapshot", leader_pid, None, true, sequence(&start_gate)?) + .await?; + let next_cycle = identity(&next_start_gate)?; + if next_cycle.0 != cycle.0 || next_cycle.1 <= cycle.1 || run.leader().await? != leader { + return Err("next-cycle assertion lost the actual coordinator or epoch".into()); + } + if fault == Some(ScopedAckFault::RequestLoss) + && (find_receiver("ack_before_apply")?.is_some() || find_receiver("ack_after_apply")?.is_some()) + { + return Err("lost scoped ACK request reached the receiver before the next cycle".into()); + } + let mut secondary_acks = Vec::new(); + for (node, peer) in producers.iter().zip(&start_producers).skip(1) { + let ack = ack_send["detail"]["acks"] + .as_array() + .ok_or("sender omitted acknowledgements")? + .iter() + .find(|ack| ack["owner_id"] == peer["owner_id"] && ack["instance_id"] == peer["instance_id"]) + .ok_or("second producer did not receive a scoped ACK")?; + let entries = tuples(&ack["entries"])?; + let original = tuples(&peer["buckets"])?; + if ack["kind"] != "scoped" + || entries.keys().cloned().collect::>() != cold + || entries.iter().any(|(bucket, tuple)| original.get(bucket) != Some(tuple)) + { + return Err("second producer ACK does not bind its own original cold tuples".into()); + } + let pid = run.pid(*node)?; + wait_until( + CYCLE_TIMEOUT, + || async { Ok(receiver_observation(&run, "ack_after_apply", pid, ack)?.is_some()) }, + "second real producer scoped ACK", + ) + .await?; + let applied = receiver_observation(&run, "ack_after_apply", pid, ack)?.ok_or("second producer omitted ACK apply")?; + let remaining = &applied["detail"]["dirty_snapshot"]; + let buckets = remaining["buckets"] + .as_array() + .ok_or("second producer omitted remaining buckets")?; + let hot_generation = tuples(&peer_for_address(&post, &run.peer_address(*node))?["buckets"])? + .get(&hot) + .ok_or("second producer omitted post-walk hot generation")? + .1; + if applied["detail"]["cleared"] != 2 + || remaining["complete"] != true + || remaining["pending_bucket_count"] != 1 + || buckets.len() != 1 + || buckets[0]["bucket"] != hot + || buckets[0]["generation"] + .as_u64() + .is_none_or(|generation| generation <= hot_generation) + { + return Err(format!("second producer did not clear cold and preserve its later hot tuple: {applied}").into()); + } + secondary_acks.push(applied); + } + // Hold the next cycle before its start map so admin reads cannot race a + // newer publication. The candidate and root evidence above are separate. + wait_until( + CYCLE_TIMEOUT, + || async { + let usage: DataUsageInfo = serde_json::from_value(run.admin_json(leader, "/rustfs/admin/v3/datausageinfo").await?)?; + Ok(usage.scanner_epoch == Some(cycle.0) && usage.scanner_cycle == Some(cycle.1)) + }, + "admin read-back of the exact complete-pending root", + ) + .await?; + let usage: DataUsageInfo = serde_json::from_value(run.admin_json(leader, "/rustfs/admin/v3/datausageinfo").await?)?; + if !usage.is_complete_bucket_usage_snapshot() + || usage.usage_snapshot_converged != Some(false) + || usage.usage_snapshot_observational + || usage.usage_snapshot_partial + || usage.usage_snapshot_bootstrap_pending + || usage.usage_snapshot_pending_buckets != vec![hot.clone()] + || usage.usage_snapshot_pending_bucket_count != 1 + || !run.usage_matches(&usage, &[&cold_a, &cold_b]) + { + return Err(format!("admin root did not retain exact cold usage and only-hot pending: {usage:?}").into()); + } + run.barrier.arm("root_durable", leader_pid, Some(next_cycle))?; + run.barrier.release("before_start_snapshot", leader_pid)?; + let next_start = run + .barrier + .wait_stage("start_snapshot", leader_pid, Some(next_cycle), false) + .await?; + let next_scope = if fault == Some(ScopedAckFault::RequestLoss) { + all.clone() + } else { + BTreeSet::from([hot.clone()]) + }; + if fault.is_some() { + let actual = tuples(&peer_for_address(&next_start, &producer_address)?["buckets"])?; + if actual.keys().cloned().collect::>() != next_scope + || (fault == Some(ScopedAckFault::RequestLoss) + && cold.iter().any(|bucket| actual.get(bucket) != start_tuples.get(bucket))) + { + return Err( + format!("next actual peer map did not retain exactly the unapplied scoped responsibility: {actual:?}").into(), + ); + } + } + let next_candidate = run + .barrier + .wait_stage("cycle_candidate", leader_pid, Some(next_cycle), false) + .await?; + assert_scope(&next_candidate, &next_scope, &start_tuples)?; + let next_root = run + .barrier + .wait_stage("root_durable", leader_pid, Some(next_cycle), true) + .await?; + let observations = run.barrier.observations(false)?; + let next_walks = observations + .iter() + .filter(|event| event["stage"].as_str() == Some("bucket_walk") && identity(event).ok() == Some(next_cycle)) + .collect::>(); + let walked_buckets = next_walks + .iter() + .map(|event| { + event["detail"]["bucket"] + .as_str() + .map(str::to_string) + .ok_or_else(|| "actual bucket walk omitted bucket".into()) + }) + .collect::>>()?; + if walked_buckets != next_scope { + return Err(format!( + "next Normal cycle must physically walk exactly the retained bucket scope {next_scope:?}: {next_walks:?}" + ) + .into()); + } + run.barrier.disarm()?; + run.barrier.release("root_durable", leader_pid)?; + let mut recovered_ack = Value::Null; + let mut recovered_usage = Value::Null; + if fault.is_some() { + let next_send = run + .barrier + .wait_stage("ack_send", leader_pid, Some(next_cycle), false) + .await?; + if sequence(&next_root)? >= sequence(&next_send)? + || next_send["detail"]["candidate_sha256"] != next_root["detail"]["candidate_sha256"] + || next_send["detail"]["coverage_sha256"] != next_root["detail"]["coverage_sha256"] + { + return Err("recovery ACK did not follow the next verified durable root".into()); + } + let retry = next_send["detail"]["acks"] + .as_array() + .ok_or("recovery cycle omitted ACKs")? + .iter() + .find(|ack| ack["owner_id"] == start_peer["owner_id"] && ack["instance_id"] == start_peer["instance_id"]) + .ok_or("recovery cycle did not acknowledge the same producer process")?; + if retry["kind"] != "scoped" + || tuples(&retry["entries"])? != tuples(&peer_for_address(&next_start, &producer_address)?["buckets"])? + { + return Err("recovery ACK did not preserve the next actual producer snapshot tuples".into()); + } + wait_until( + CYCLE_TIMEOUT, + || async { Ok(receiver_observation(&run, "ack_after_apply", producer_pid, retry)?.is_some()) }, + "actual scoped ACK recovery after transport loss", + ) + .await?; + recovered_ack = + receiver_observation(&run, "ack_after_apply", producer_pid, retry)?.ok_or("recovery receiver omitted apply")?; + let remaining = &recovered_ack["detail"]["dirty_snapshot"]; + if recovered_ack["detail"]["cleared"].as_u64() != Some(u64::try_from(next_scope.len())?) + || remaining["complete"] != true + || remaining["pending_bucket_count"] != 0 + || remaining["buckets"].as_array().is_none_or(|entries| !entries.is_empty()) + { + return Err(format!("healthy recovery did not clear precisely the retained responsibility: {recovered_ack}").into()); + } + recovered_usage = serde_json::to_value(run.quiet_baseline().await?)?; + } + run.barrier.stop_session()?; + run.verify_objects().await?; + run.write_evidence( + label, + &json!({"topology": run.topology, "objects": run.objects, "baseline": baseline, "scheduler_config": scheduler_config, + "leader_node": leader, "leader_pid": leader_pid, "producer_node": producer, "producer_pid": producer_pid, + "producers": producers, "start_producers": start_producers, "secondary_acks": secondary_acks, + "start_cycle_wake": start_cycle_wake, "next_cycle_wake": next_cycle_wake, + "hot_in_post_snapshot_window": hot_in_post_snapshot_window[0], "hot_after_root": hot_after_root[0], + "hot_in_post_snapshot_window_by_producer": hot_in_post_snapshot_window, "hot_after_root_by_producer": hot_after_root, + "hot_stream": hot_stream, "hot_stream_completed_at_root": completed_at_root, + "cycle": cycle, "next_cycle": next_cycle, "usage": usage, + "candidate": candidate, "root": root, "ack_send": ack_send, "ack_before_apply": before_ack, + "ack_after_apply": after_ack, "fault_receipt": fault_receipt, "next_scope": next_scope, + "next_start": next_start, "next_candidate": next_candidate, "next_root": next_root, + "recovered_ack": recovered_ack, "recovered_usage": recovered_usage, "observations": run.barrier.observations(false)?}), + ) +} diff --git a/crates/e2e_test/src/rpc_fault_proxy.rs b/crates/e2e_test/src/rpc_fault_proxy.rs index 042757630..930eaa6ae 100644 --- a/crates/e2e_test/src/rpc_fault_proxy.rs +++ b/crates/e2e_test/src/rpc_fault_proxy.rs @@ -23,7 +23,7 @@ use hyper_util::{ server::conn::auto, }; use prost::Message as _; -use rustfs_protos::proto_gen::node_service::{DeleteRequest, ReadVersionRequest}; +use rustfs_protos::proto_gen::node_service::{DeleteRequest, ReadVersionRequest, ScannerScopedDirtyUsageAckRequest}; use std::{ collections::HashSet, io, @@ -43,6 +43,9 @@ use tokio::{ type ProxyBody = BoxBody; type ProxyClient = Client; +mod scoped_ack; +pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope}; + #[derive(Clone, Copy, PartialEq, Eq)] enum Disposition { Hold, @@ -74,7 +77,8 @@ impl RpcPause { self.state.blocked.load(Ordering::SeqCst) } - /// Selected RPCs whose backend returned a response, excluding discarded packets. + /// Selected RPCs whose backend returned response headers, excluding + /// discarded packets. This does not prove a successful gRPC trailer or ACK apply. pub(crate) fn forwarded(&self) -> usize { self.state.forwarded.load(Ordering::SeqCst) } @@ -132,6 +136,7 @@ impl Drop for RpcPause { pub(crate) struct RpcFaultProxy { address: SocketAddr, slot: PauseSlot, + scoped_ack: scoped_ack::QueueSlot, task: JoinHandle<()>, } @@ -141,6 +146,8 @@ impl RpcFaultProxy { let address = listener.local_addr()?; let slot: PauseSlot = Arc::new(Mutex::new(None)); let active = slot.clone(); + let scoped_ack = scoped_ack::empty_slot(); + let active_ack = scoped_ack.clone(); let task = tokio::spawn(async move { let h1: ProxyClient = Client::builder(TokioExecutor::new()).build_http(); let h2: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); @@ -152,11 +159,13 @@ impl RpcFaultProxy { let h1 = h1.clone(); let h2 = h2.clone(); let active = active.clone(); + let active_ack = active_ack.clone(); connections.spawn(async move { let service = service_fn(move |request: Request| { let client = if request.version() == http::Version::HTTP_2 { h2.clone() } else { h1.clone() }; let pause = active.lock().expect("RPC pause slot").clone(); - forward_request(request, client, target, pause) + let ack = active_ack.lock().expect("scoped ACK queue slot").clone(); + forward_request(request, client, target, pause, ack) }); let _ = auto::Builder::new(TokioExecutor::new()) @@ -167,13 +176,23 @@ impl RpcFaultProxy { } } }); - Ok(Self { address, slot, task }) + Ok(Self { + address, + slot, + scoped_ack, + task, + }) } pub(crate) fn local_addr(&self) -> SocketAddr { self.address } + /// Retain only genuine scoped ACKs for one receiver/session and bucket set. + pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result { + ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity) + } + /// Pause at most limit requests to the named unary methods; other streams /// on the same HTTP/2 connection continue forwarding. pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result { @@ -217,6 +236,9 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> { } fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result { + if method == "ScannerScopedDirtyUsageAck" { + return scoped_ack_matches(body, Some(scope)); + } if method != "Delete" { return Ok(body.windows(scope.len()).any(|window| window == scope)); } @@ -227,26 +249,50 @@ fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result { && options.get("recursive").and_then(serde_json::Value::as_bool) == Some(true)) } +fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result { + let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?; + Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope))) +} + async fn forward_request( request: Request, client: ProxyClient, target: SocketAddr, pause: Option>, -) -> io::Result> { - let (mut parts, body) = request.into_parts(); - let mut body = body.map_err(io::Error::other).boxed(); + ack: Option>, +) -> io::Result> { + let request = request.map(|body| body.map_err(io::Error::other).boxed()); + let request = if let Some(ack) = ack + && request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" + { + match scoped_ack::intercept(request, client.clone(), target, ack).await? { + scoped_ack::Interception::Pass(request) => request, + scoped_ack::Interception::Handled(response) => return Ok(response), + } + } else { + request + }; + let (parts, body) = request.into_parts(); + let mut body = body; let mut observed = None; if let Some(pause) = pause { - let method = parts.uri.path().strip_prefix("/node_service.NodeService/"); + let method = parts + .uri + .path() + .strip_prefix("/node_service.NodeService/") + .or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/")); if let Some(method) = method.filter(|name| pause.methods.contains(name)) { let mut route = None; - let matches = if let Some(scope) = pause.scope.as_deref() { + let matches = if pause.scope.is_some() || method == "ScannerScopedDirtyUsageAck" { // Inspect bounded control routing while preserving the exact // signed bytes and trailers forwarded to the backend. let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?; let trailers = collected.trailers().cloned(); let bytes = collected.to_bytes(); - let matches = scope_matches(method, &bytes, scope)?; + let matches = match pause.scope.as_deref() { + Some(scope) => scope_matches(method, &bytes, scope)?, + None => scoped_ack_matches(&bytes, None)?, + }; if matches && method == "ReadVersion" { let disk = ReadVersionRequest::decode(unary_payload(&bytes)?) .map_err(io::Error::other)? @@ -283,6 +329,25 @@ async fn forward_request( } } } + let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?; + if let Some((pause, route)) = observed + && response.status().is_success() + && response.headers().get("grpc-status").is_none_or(|status| status == "0") + { + pause.forwarded.fetch_add(1, Ordering::SeqCst); + if let Some(route) = route { + pause.forwarded_routes.lock().expect("RPC route observations").insert(route); + } + } + Ok(response.map(|body| body.map_err(io::Error::other).boxed())) +} + +async fn forward_to_backend( + request: Request, + client: ProxyClient, + target: SocketAddr, +) -> io::Result> { + let (mut parts, body) = request.into_parts(); if !parts.headers.contains_key(http::header::HOST) && let Some(authority) = parts.uri.authority() { @@ -297,20 +362,10 @@ async fn forward_request( .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str())) .build() .map_err(io::Error::other)?; - let response = client + client .request(Request::from_parts(parts, body)) .await - .map_err(io::Error::other)?; - if let Some((pause, route)) = observed - && response.status().is_success() - && response.headers().get("grpc-status").is_none_or(|status| status == "0") - { - pause.forwarded.fetch_add(1, Ordering::SeqCst); - if let Some(route) = route { - pause.forwarded_routes.lock().expect("RPC route observations").insert(route); - } - } - Ok(response) + .map_err(io::Error::other) } impl Drop for RpcFaultProxy { @@ -327,6 +382,7 @@ mod tests { struct Backend { received: Arc, + auth_headers: Arc>>, address: SocketAddr, task: JoinHandle<()>, } @@ -342,6 +398,8 @@ mod tests { let address = listener.local_addr()?; let received = Arc::new(AtomicUsize::new(0)); let received_task = received.clone(); + let auth_headers = Arc::new(Mutex::new(Vec::new())); + let backend_auth = auth_headers.clone(); let task = tokio::spawn(async move { let mut children = JoinSet::new(); loop { @@ -349,12 +407,18 @@ mod tests { accepted = listener.accept() => { let Ok((socket, _)) = accepted else { break }; let received = received_task.clone(); + let backend_auth = backend_auth.clone(); children.spawn(async move { let service = service_fn(move |request: Request| { let received = received.clone(); + let backend_auth = backend_auth.clone(); async move { received.fetch_add(1, Ordering::SeqCst); let authority = request.headers().get(http::header::HOST).cloned(); + if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) { + backend_auth.lock().expect("backend auth header observations").push(( + nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string())); + } let body = request.into_body().collect().await?.to_bytes(); let mut trailers = http::HeaderMap::new(); trailers.insert("grpc-status", http::HeaderValue::from_static("0")); @@ -375,7 +439,12 @@ mod tests { } } }); - Ok(Backend { received, address, task }) + Ok(Backend { + received, + auth_headers, + address, + task, + }) } async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result> { @@ -389,15 +458,367 @@ mod tests { bytes: Bytes, ) -> io::Result> { let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed(); + let service = if method == "ScannerScopedDirtyUsageAck" { + "ScannerControlService" + } else { + "NodeService" + }; let request = Request::builder() .method("POST") - .uri(format!("http://{address}/node_service.NodeService/{method}")) + .uri(format!("http://{address}/node_service.{service}/{method}")) .header(http::header::TE, "trailers") .body(body) .map_err(io::Error::other)?; client.request(request).await.map_err(io::Error::other) } + const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111"; + const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn ack_scope() -> ScopedAckScope { + ScopedAckScope { + nonce: uuid::Uuid::new_v4().to_string(), + owner_id: ACK_OWNER.to_string(), + instance_id: ACK_INSTANCE.to_string(), + buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]), + } + } + + fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest { + ScannerScopedDirtyUsageAckRequest { + challenge: vec![7; 16].into(), + protocol_version: 1, + owner_id: ACK_OWNER.to_string(), + instance_id: ACK_INSTANCE.to_string(), + scope: 1, + probe_only: false, + entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry { + bucket: bucket.to_string(), + bucket_incarnation: vec![4; 16].into(), + generation, + }], + } + } + + fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes { + let payload = request.encode_to_vec(); + let mut frame = vec![0]; + frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes()); + frame.extend_from_slice(&payload); + Bytes::from(frame) + } + + fn ack_hash(bytes: &[u8]) -> String { + use sha2::Digest as _; + hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower) + } + + // Transport-only fixtures: the distributed tests supply production signatures. + async fn call_ack_frame( + client: &ProxyClient, + address: SocketAddr, + frame: Bytes, + nonce: &str, + ) -> io::Result> { + let request = Request::builder() + .method("POST") + .uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck")) + .header(http::header::TE, "trailers") + .header("x-rustfs-rpc-nonce", nonce) + .header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}")) + .body(Full::new(frame).map_err(|never| match never {}).boxed()) + .map_err(io::Error::other)?; + client.request(request).await.map_err(io::Error::other) + } + + #[tokio::test] + async fn scoped_ack_queue_loss_preserves_backend_boundary() { + tokio::time::timeout(Duration::from_secs(10), async { + for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] { + let backend = backend().await.expect("backend"); + let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address) + .await + .expect("proxy"); + let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue"); + let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); + let frame = ack_frame(&ack_request("cold-bucket", 7)); + let address = proxy.local_addr(); + let body = frame.clone(); + let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await }); + queue.wait_for_packets(1).await.expect("captured ACK"); + assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch"); + queue.settle(0, fault).expect("settle actual retained packet"); + assert!( + caller.await.expect("caller task").is_err(), + "the caller must observe the injected transport failure" + ); + let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt"); + let response_loss = fault == ScopedAckFault::ResponseLoss; + assert_eq!(receipt["forwarded"], response_loss); + assert_eq!(receipt["backend_complete"], response_loss); + assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss)); + if response_loss { + assert_eq!(receipt["outcome"], "response_lost"); + assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers"); + assert_eq!(receipt["response"]["bytes"], frame.len()); + assert_eq!(receipt["response"]["sha256"], ack_hash(&frame)); + assert!( + receipt["response"]["trailers_sha256"] + .as_str() + .is_some_and(|value| value.len() == 64) + ); + } else { + assert_eq!(receipt["outcome"], "request_lost"); + } + } + }) + .await + .expect("bounded ACK loss checks"); + } + + #[tokio::test] + async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() { + tokio::time::timeout(Duration::from_secs(10), async { + let backend = backend().await.expect("backend"); + let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address) + .await + .expect("proxy"); + let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue"); + let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); + let first = ack_frame(&ack_request("cold-bucket", 7)); + let second = ack_frame(&ack_request("cold-bucket", 9)); + let address = proxy.local_addr(); + let old_client = client.clone(); + let old_body = first.clone(); + let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await }); + queue.wait_for_packets(1).await.expect("first retained packet"); + old_caller.abort(); + assert!(old_caller.await.expect_err("cancel original caller").is_cancelled()); + let new_body = second.clone(); + let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await }); + queue.wait_for_packets(2).await.expect("second retained packet"); + assert_eq!(backend.received.load(Ordering::SeqCst), 0); + queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first"); + let response = new_caller + .await + .expect("new caller task") + .expect("new response") + .into_body() + .collect() + .await + .expect("complete body"); + assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof"); + assert_eq!(response.to_bytes(), second); + queue.wait_for_completion(1).await.expect("new packet complete"); + queue + .settle(0, ScopedAckFault::Forward) + .expect("release old packet after its caller disappeared"); + let old = queue.wait_for_completion(0).await.expect("detached old packet complete"); + assert_eq!(old["backend_complete"], true); + assert_eq!(old["request_sha256"], ack_hash(&first)); + assert_eq!(old["response"]["sha256"], ack_hash(&first)); + assert_eq!(backend.received.load(Ordering::SeqCst), 2); + assert_eq!( + *backend.auth_headers.lock().expect("backend header observations"), + vec![ + ("second".to_string(), "opaque-signature-second".to_string()), + ("first".to_string(), "opaque-signature-first".to_string()) + ] + ); + assert!( + queue.settle(0, ScopedAckFault::Forward).is_err(), + "one original packet may only be forwarded once" + ); + }) + .await + .expect("bounded detached retention check"); + } + + #[tokio::test] + async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() { + tokio::time::timeout(Duration::from_secs(10), async { + let backend = backend().await.expect("backend"); + let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address) + .await + .expect("proxy"); + assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err()); + assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err()); + let mut invalid = ack_scope(); + invalid.nonce.clear(); + assert!(proxy.scoped_ack_queue(invalid, 1).is_err()); + let mut scope = ack_scope(); + scope.buckets.insert("cold-bucket-z".to_string()); + let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue"); + let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); + let mut probe = ack_request("cold-bucket", 7); + probe.probe_only = true; + let mut other_peer = ack_request("cold-bucket", 7); + other_peer.owner_id = uuid::Uuid::new_v4().to_string(); + let mut other_instance = ack_request("cold-bucket", 7); + other_instance.instance_id = "b".repeat(32); + let mut invalid_challenge = ack_request("cold-bucket", 7); + invalid_challenge.challenge = vec![7; 15].into(); + let mut noncanonical_entries = ack_request("cold-bucket-z", 7); + noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries); + let unselected = [ + probe, + other_peer, + other_instance, + ack_request("cold-bucket-other", 7), + ack_request("cold-bucket", u64::MAX), + invalid_challenge, + noncanonical_entries, + ]; + let unselected_count = unselected.len(); + for (index, request) in unselected.into_iter().enumerate() { + let body = ack_frame(&request); + let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected") + .await + .expect("unselected ACK progresses"); + assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body); + assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot"); + assert_eq!(backend.received.load(Ordering::SeqCst), index + 1); + } + assert!(queue.receipts().is_empty()); + let address = proxy.local_addr(); + let held_client = client.clone(); + let valid_frame = ack_frame(&ack_request("cold-bucket", 7)); + let held_body = valid_frame.clone(); + let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await }); + queue.wait_for_packets(1).await.expect("one matching packet"); + assert!(!held.is_finished(), "the subsequent valid ACK must be retained"); + assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame)); + assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count); + let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity") + .await + .expect("capacity overflow passes without retention"); + extra.into_body().collect().await.expect("overflow response"); + assert_eq!(queue.receipts().len(), 1); + assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1); + drop(queue); + assert!( + held.await.expect("held caller").is_err(), + "dropping the session must cancel its retained packet" + ); + assert_eq!( + backend.received.load(Ordering::SeqCst), + unselected_count + 1, + "session drop must never forward the held packet" + ); + }) + .await + .expect("bounded scope and lifecycle checks"); + } + + #[tokio::test] + async fn scoped_ack_queue_expiry_never_forwards_retained_packet() { + let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async { + let backend = backend().await.expect("backend"); + let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address) + .await + .expect("proxy"); + let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue"); + let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); + let address = proxy.local_addr(); + let caller = tokio::spawn(async move { + call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await + }); + queue.wait_for_packets(1).await.expect("retained packet"); + (backend, proxy, queue, caller) + }) + .await + .expect("bounded queue setup"); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(120)).await; + tokio::time::resume(); + assert!( + tokio::time::timeout(Duration::from_secs(10), caller) + .await + .expect("expiry completes caller") + .expect("caller task") + .is_err() + ); + let receipt = queue.receipts().into_iter().next().expect("expired receipt"); + assert_eq!(receipt["completed"], true); + assert_eq!(receipt["forwarded"], false); + assert_eq!(receipt["backend_complete"], false); + assert_eq!(backend.received.load(Ordering::SeqCst), 0); + assert_eq!( + queue + .settle(0, ScopedAckFault::Forward) + .expect_err("expired packet cannot be released") + .kind(), + io::ErrorKind::TimedOut + ); + } + + #[tokio::test] + async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() { + use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry; + + tokio::time::timeout(Duration::from_secs(10), async { + let backend = backend().await.expect("backend listener"); + let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address) + .await + .expect("proxy listener"); + let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http(); + let pause = proxy + .pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket")) + .expect("scoped ACK fault"); + let frame = |bucket: &str, probe_only: bool| { + let payload = ScannerScopedDirtyUsageAckRequest { + challenge: vec![3; 16].into(), + protocol_version: 1, + owner_id: "11111111-1111-1111-1111-111111111111".to_string(), + instance_id: "a".repeat(32), + scope: 1, + probe_only, + entries: vec![ScannerScopedDirtyUsageEntry { + bucket: bucket.to_string(), + bucket_incarnation: vec![4; 16].into(), + generation: 17, + }], + } + .encode_to_vec(); + let mut frame = vec![0]; + frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes()); + frame.extend_from_slice(&payload); + Bytes::from(frame) + }; + for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] { + let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone()) + .await + .expect("probe and other bucket must progress"); + assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes); + } + assert_eq!(pause.hits(), 0); + let signed_bytes = frame("cold-bucket", false); + let task_bytes = signed_bytes.clone(); + let address = proxy.local_addr(); + let blocked = + tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await }); + pause + .wait_for_hits(1) + .await + .expect("real ScannerControlService method must reach the fault"); + assert!(!blocked.is_finished()); + assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend"); + pause.release(); + let collected = blocked + .await + .expect("ACK task") + .expect("released ACK") + .into_body() + .collect() + .await + .expect("ACK body and trailers"); + assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof"); + assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame"); + }) + .await + .expect("scanner ACK fault deadline"); + } + #[tokio::test] async fn preserves_protocol_authority_payload_and_grpc_trailers() { tokio::time::timeout(Duration::from_secs(10), async { diff --git a/crates/e2e_test/src/rpc_fault_proxy/scoped_ack.rs b/crates/e2e_test/src/rpc_fault_proxy/scoped_ack.rs new file mode 100644 index 000000000..cec0d8c30 --- /dev/null +++ b/crates/e2e_test/src/rpc_fault_proxy/scoped_ack.rs @@ -0,0 +1,404 @@ +// Copyright 2026 RustFS Team +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded retention of original scoped ACK frames, independent of caller cancellation. + +use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload}; +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full, Limited}; +use hyper::{Request, Response}; +use prost::Message as _; +use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse}; +use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request; +use serde_json::{Value, json}; +use sha2::{Digest as _, Sha256}; +use std::{ + collections::BTreeSet, + io, + net::SocketAddr, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use tokio::{ + sync::{Notify, oneshot, watch}, + task::AbortHandle, + time::Instant, +}; + +const MAX_RETAINED: usize = 4; +const MAX_BYTES: usize = 1024 * 1024; +const RETENTION: Duration = Duration::from_secs(120); + +pub(crate) struct ScopedAckScope { + pub nonce: String, + pub owner_id: String, + pub instance_id: String, + pub buckets: BTreeSet, +} + +impl ScopedAckScope { + fn validate(&self) -> io::Result<()> { + let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil()); + if !valid_uuid(&self.nonce) + || !valid_uuid(&self.owner_id) + || self.instance_id.is_empty() + || self.instance_id.len() > 128 + || self.buckets.is_empty() + || self.buckets.len() > 16 + || self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "scoped ACK queue needs a bounded session, receiver and bucket scope", + )); + } + Ok(()) + } + + fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool { + !request.probe_only + && validate_scoped_dirty_usage_request(request).is_ok() + && request.owner_id == self.owner_id + && request.instance_id == self.instance_id + && request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScopedAckFault { + Forward, + RequestLoss, + ResponseLoss, +} + +struct Packet { + action: watch::Sender>, + receipt: Mutex, + task: Mutex>, +} + +pub(super) struct QueueState { + scope: ScopedAckScope, + capacity: usize, + started: Instant, + deadline: Instant, + closed: AtomicBool, + // The packet inventory lock is released before any packet lock or await. + packets: Mutex>>, + changed: Notify, +} + +pub(super) type QueueSlot = Arc>>>; + +pub(super) fn empty_slot() -> QueueSlot { + Arc::new(Mutex::new(None)) +} + +pub(crate) struct ScopedAckQueue { + state: Arc, + slot: QueueSlot, +} + +impl ScopedAckQueue { + pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result { + scope.validate()?; + if !(1..=MAX_RETAINED).contains(&capacity) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4")); + } + let mut active = slot.lock().expect("scoped ACK queue slot"); + if active.is_some() { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed")); + } + let started = Instant::now(); + let state = Arc::new(QueueState { + scope, + capacity, + started, + deadline: started + RETENTION, + closed: AtomicBool::new(false), + packets: Mutex::new(Vec::new()), + changed: Notify::new(), + }); + *active = Some(state.clone()); + drop(active); + Ok(Self { state, slot }) + } + + pub(crate) fn receipts(&self) -> Vec { + let packets = self.state.packets.lock().expect("scoped ACK packets").clone(); + packets + .iter() + .map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone()) + .collect() + } + + pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> { + self.wait_until(|| self.receipts().len() >= count).await + } + + pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result { + self.wait_until(|| { + self.receipts() + .get(id) + .is_some_and(|receipt| receipt["completed"].as_bool() == Some(true)) + }) + .await?; + self.receipts() + .get(id) + .cloned() + .ok_or_else(|| io::Error::other("scoped ACK receipt disappeared")) + } + + async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> { + tokio::time::timeout_at(self.state.deadline, async { + loop { + let changed = self.state.changed.notified(); + if ready() { + return; + } + changed.await; + } + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed")) + } + + pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> { + if Instant::now() >= self.state.deadline { + return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")); + } + let packet = self + .state + .packets + .lock() + .expect("scoped ACK packets") + .get(id) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?; + if !packet.action.send_if_modified(|current| { + if current.is_none() { + *current = Some(fault); + true + } else { + false + } + }) { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled")); + } + Ok(()) + } +} + +impl Drop for ScopedAckQueue { + fn drop(&mut self) { + self.state.closed.store(true, Ordering::SeqCst); + let mut slot = self.slot.lock().expect("scoped ACK queue slot"); + if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) { + *slot = None; + } + drop(slot); + let packets = self.state.packets.lock().expect("scoped ACK packets").clone(); + for packet in packets { + if let Some(task) = packet.task.lock().expect("scoped ACK task").take() { + task.abort(); + } + } + } +} + +pub(super) enum Interception { + Pass(Request), + Handled(Response), +} + +fn restored_body(bytes: Bytes, trailers: Option) -> ProxyBody { + Full::new(bytes) + .map_err(|never| match never {}) + .with_trailers(async move { trailers.map(Ok::<_, io::Error>) }) + .boxed() +} + +fn hash(bytes: &[u8]) -> String { + hex(Sha256::digest(bytes).as_ref()) +} + +fn hex(bytes: &[u8]) -> String { + hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower) +} + +fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value { + trailers.map_or(Value::Null, |headers| { + json!( + headers + .iter() + .map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) }) + .collect::>() + ) + }) +} + +fn header_hash(headers: &http::HeaderMap) -> String { + let mut hasher = Sha256::new(); + for (name, value) in headers { + hasher.update(name.as_str().as_bytes()); + hasher.update([0]); + hasher.update(value.as_bytes()); + hasher.update([0]); + } + hex(hasher.finalize().as_ref()) +} + +fn millis(state: &QueueState) -> u64 { + u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +pub(super) async fn intercept( + request: Request, + client: ProxyClient, + target: SocketAddr, + state: Arc, +) -> io::Result { + if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) { + return Ok(Interception::Pass(request)); + } + let (parts, body) = request.into_parts(); + let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect()) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))? + .map_err(io::Error::other)?; + let trailers = collected.trailers().cloned(); + let bytes = collected.to_bytes(); + let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?; + let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone())); + if Instant::now() >= state.deadline || !state.scope.matches(&decoded) { + return Ok(Interception::Pass(request)); + } + let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok()); + let (action, mut selected) = watch::channel(None); + let packet = { + let mut packets = state.packets.lock().expect("scoped ACK packets"); + if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) { + return Ok(Interception::Pass(request)); + } + let id = packets.len(); + let packet = Arc::new(Packet { + action, + task: Mutex::new(None), + receipt: Mutex::new(json!({ + "id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(), + "authority": request.uri().authority().map(|value| value.as_str()), + "captured_at_ms": millis(&state), "completed": false, "forwarded": false, + "backend_complete": false, "outcome": "held", "request_bytes": bytes.len(), + "request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()), + "request_trailers_sha256": trailers.as_ref().map(header_hash), + "request_trailers": trailer_receipt(trailers.as_ref()), + "auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"), + "signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())), + "replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())), + "request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id, + "probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({ + "bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()), + "generation": entry.generation })).collect::>() } + })), + }); + packets.push(packet.clone()); + packet + }; + let (reply, response) = oneshot::channel(); + let worker_packet = packet.clone(); + let worker_state = state.clone(); + // This owner survives an HTTP/2 stream reset from the original 5 s caller. + // Every request is settled once and is discarded at the session deadline. + let task = tokio::spawn(async move { + let result = tokio::time::timeout_at(worker_state.deadline, async { + let fault = loop { + if let Some(fault) = *selected.borrow_and_update() { + break fault; + } + selected + .changed() + .await + .map_err(|_| io::Error::other("scoped ACK queue closed"))?; + }; + if fault == ScopedAckFault::RequestLoss { + worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost"); + return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss")); + } + if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline { + return Err(io::Error::new( + io::ErrorKind::ConnectionAborted, + "scoped ACK session ended or expired before forwarding", + )); + } + { + let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt"); + receipt["forwarded"] = json!(true); + receipt["forwarded_at_ms"] = json!(millis(&worker_state)); + } + let backend = forward_to_backend(request, client, target).await?; + let (parts, body) = backend.into_parts(); + let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?; + let trailers = collected.trailers().cloned(); + let bytes = collected.to_bytes(); + let grpc_status = trailers + .as_ref() + .and_then(|values| values.get("grpc-status")) + .or_else(|| parts.headers.get("grpc-status")) + .and_then(|value| value.to_str().ok()); + let response = unary_payload(&bytes) + .ok() + .and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok()) + .map(|response| { + json!({"owner_id": response.owner_id, "instance_id": response.instance_id, + "supported": response.supported, "cleared": response.cleared, + "response_proof_bytes": response.response_proof.len()}) + }); + { + let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt"); + receipt["backend_complete"] = json!(true); + receipt["backend_completed_at_ms"] = json!(millis(&worker_state)); + receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status, + "bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers), + "trailers": trailer_receipt(trailers.as_ref()), + "trailers_sha256": trailers.as_ref().map(header_hash), "ack": response}); + receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss { + "response_lost" + } else { + "forwarded" + }); + } + if fault == ScopedAckFault::ResponseLoss { + return Err(io::Error::new( + io::ErrorKind::ConnectionAborted, + "injected scoped ACK response loss after backend trailers", + )); + } + Ok(Response::from_parts(parts, restored_body(bytes, trailers))) + }) + .await + .unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"))); + { + let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt"); + if result.is_err() && receipt["outcome"] == "held" { + receipt["outcome"] = json!("transport_failed_or_expired"); + } + receipt["completed"] = json!(true); + receipt["completed_at_ms"] = json!(millis(&worker_state)); + } + let _ = reply.send(result); + worker_state.changed.notify_one(); + }); + *packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle()); + if state.closed.load(Ordering::SeqCst) { + task.abort(); + } + drop(task); + state.changed.notify_one(); + response + .await + .map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))? + .map(Interception::Handled) +} diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 0205328c6..9f47f8666 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -31,6 +31,7 @@ workspace = true [features] default = [] +e2e-test-hooks = [] diagnostic-read-version-caller = ["rustfs-ecstore/diagnostic-read-version-caller"] hotpath = [ "hotpath/hotpath", diff --git a/crates/scanner/src/e2e_barrier.rs b/crates/scanner/src/e2e_barrier.rs new file mode 100644 index 000000000..c39d465c8 --- /dev/null +++ b/crates/scanner/src/e2e_barrier.rs @@ -0,0 +1,209 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test-cluster observations and bounded pauses at real scanner boundaries. +//! This module is absent unless `e2e-test-hooks` is explicitly enabled. + +use serde::Deserialize; +use serde_json::{Value, json}; +use std::io::{self, Write}; +use std::path::{Component, Path}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +const MAX_BYTES: u64 = 1024 * 1024; +const MAX_EVENTS: u64 = 512; +const MAX_WAIT: Duration = Duration::from_secs(120); +static SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Session { + nonce: String, + buckets: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Arm { + nonce: String, + stage: String, + pid: u32, + owner_id: Option, + instance_id: Option, + epoch: Option, + cycle: Option, +} + +fn valid_component(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') +} + +fn read_control(path: &Path) -> io::Result> { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if !metadata.is_file() || metadata.len() > MAX_BYTES { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid scanner test control file")); + } + let data = std::fs::read(path)?; + if data.len() > MAX_BYTES as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "scanner test control file grew past its bound", + )); + } + serde_json::from_slice(&data).map(Some).map_err(io::Error::other) +} + +fn write_observation(directory: &Path, name: &str, bytes: &[u8]) -> io::Result<()> { + let temporary = directory.join(format!(".{name}.tmp")); + let mut output = std::fs::OpenOptions::new().write(true).create_new(true).open(&temporary)?; + output.write_all(bytes)?; + output.sync_all()?; + std::fs::rename(temporary, directory.join(name)) +} + +/// Observe a real event and, only for an explicitly armed matching session, +/// wait for its release file. No test control can supply scanner state. +pub async fn checkpoint(stage: &str, detail: Value) -> io::Result<()> { + let Some(directory) = std::env::var_os("RUSTFS_E2E_SCANNER_BARRIER_DIR") else { + return Ok(()); + }; + let directory = Path::new(&directory); + if !directory.is_absolute() + || directory.components().any(|part| matches!(part, Component::ParentDir)) + || !valid_component(stage) + { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid scanner test barrier path or stage")); + } + let Some(session) = read_control::(&directory.join("session.json"))? else { + return Ok(()); + }; + if !valid_component(&session.nonce) + || session.buckets.is_empty() + || session.buckets.len() > 32 + || session.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 255) + { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid scanner test session")); + } + if detail + .get("bucket") + .and_then(Value::as_str) + .is_some_and(|bucket| !session.buckets.iter().any(|allowed| allowed == bucket)) + { + return Ok(()); + } + let arm = read_control::(&directory.join("arm.json"))?; + let armed = arm.is_some_and(|arm| { + arm.nonce == session.nonce + && arm.stage == stage + && arm.pid == std::process::id() + && arm + .owner_id + .as_ref() + .is_none_or(|owner| detail.get("owner_id").and_then(Value::as_str) == Some(owner.as_str())) + && arm + .instance_id + .as_ref() + .is_none_or(|instance| detail.get("instance_id").and_then(Value::as_str) == Some(instance.as_str())) + && arm + .epoch + .is_none_or(|epoch| detail.get("epoch").and_then(Value::as_u64) == Some(epoch)) + && arm + .cycle + .is_none_or(|cycle| detail.get("cycle").and_then(Value::as_u64) == Some(cycle)) + }); + let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed); + if sequence >= MAX_EVENTS { + return Err(io::Error::other("scanner test observation limit reached")); + } + let pid = std::process::id(); + let event = json!({ + "nonce": session.nonce, + "pid": pid, + "sequence": sequence, + "stage": stage, + "armed": armed, + "detail": detail, + }); + let encoded = serde_json::to_vec(&event).map_err(io::Error::other)?; + if encoded.len() > MAX_BYTES as usize { + return Err(io::Error::other("scanner test observation exceeds its byte bound")); + } + let suffix = format!("{}.{}.{}.json", session.nonce, pid, sequence); + write_observation(directory, &format!("observed.{suffix}"), &encoded)?; + if !armed { + return Ok(()); + } + write_observation(directory, &format!("reached.{suffix}"), &encoded)?; + let release = directory.join(format!("release.{}.{}.{}", session.nonce, stage, pid)); + let started = Instant::now(); + while !release.try_exists()? { + if started.elapsed() >= MAX_WAIT { + return Err(io::Error::new(io::ErrorKind::TimedOut, "scanner test barrier release timed out")); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Ok(()) +} + +pub(crate) fn hex(bytes: &[u8]) -> String { + hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower) +} + +pub(crate) fn acknowledgement_observations(acknowledgements: &[crate::scanner::ScannerDirtyUsageAcknowledgement]) -> Value { + Value::Array( + acknowledgements + .iter() + .map(|acknowledgement| { + use crate::scanner::ScannerDirtyUsageAcknowledgementKind; + match &acknowledgement.kind { + ScannerDirtyUsageAcknowledgementKind::Generation(generation) => json!({ + "host": acknowledgement.host, + "instance_id": acknowledgement.instance_id, + "kind": "generation", + "generation": generation, + }), + ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } => json!({ + "host": acknowledgement.host, + "owner_id": owner_id, + "instance_id": acknowledgement.instance_id, + "kind": "scoped", + "entries": entries.iter().map(|entry| json!({ + "bucket": entry.bucket, + "bucket_incarnation": entry.bucket_incarnation, + "generation": entry.generation, + })).collect::>(), + }), + } + }) + .collect(), + ) +} + +pub(crate) fn activity_observations(activity: &crate::scanner::ScannerActivitySnapshot) -> Value { + Value::Object( + activity + .iter() + .map(|(host, activity)| (host.clone(), activity.e2e_observation())) + .collect(), + ) +} diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 973f7282f..7559d626e 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -58,6 +58,9 @@ use tokio_util::sync::CancellationToken; use uuid::Uuid; pub mod data_usage_define; +#[cfg(feature = "e2e-test-hooks")] +#[doc(hidden)] +pub mod e2e_barrier; pub mod error; mod heal_completion; pub mod prefix_usage; diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 8c5a1ceb2..a3889ca84 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -792,6 +792,16 @@ async fn scan_and_persist_local_bucket( let set_disks = scanner_set_disk_inventory(set.as_ref()).await; let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::(1); let scan_ctx = ctx.child_token(); + #[cfg(feature = "e2e-test-hooks")] + let _ = crate::e2e_barrier::checkpoint( + "bucket_walk", + serde_json::json!({ + "epoch": leader_epoch, "cycle": next_cycle, + "pool": source.pool_index, "set": source.set_index, + "bucket": bucket, "drive": disk.path().display().to_string(), + }), + ) + .await; let scan = ScannerIODisk::nsscanner_disk( disk.clone(), scan_ctx.clone(), diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index f757bf2a7..b88747b3b 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -2024,6 +2024,8 @@ where resolved_scope_observer: None, #[cfg(test)] post_walk_gate: None, + #[cfg(test)] + peer_observations: None, }, ) .await; @@ -2164,6 +2166,9 @@ where scanner_post_lease_activity_defer_reason( result.activity_baseline(), probe_scanner_activity(storeapi.as_ref(), true).await, + result + .remote_dirty_usage_certification() + .zip(remote_publication_leases.as_ref().map(|(_, grants)| grants.as_slice())), ) } else { None @@ -2515,9 +2520,24 @@ where } else { "incomplete_coverage" }; + #[cfg(feature = "e2e-test-hooks")] + let durable_observation = scan_cycle_result + .publication_expectation() + .as_ref() + .and_then(|expected| usage_publication_result.e2e_durable_observation(leader_epoch, cycle_info.current, expected)); + #[cfg(feature = "e2e-test-hooks")] + if let Some(observation) = &durable_observation { + let _ = crate::e2e_barrier::checkpoint("root_durable", observation.clone()).await; + } let pending_dirty_usage_buckets = scan_cycle_result.pending_dirty_usage_buckets(); let (completion_outcome, scanner_pending_maintenance_work, remote_dirty_usage_acknowledgements) = finalize_scanner_cycle_result(scan_cycle_result, usage_publication_result); + #[cfg(feature = "e2e-test-hooks")] + if let Some(mut observation) = durable_observation { + observation["acks"] = crate::e2e_barrier::acknowledgement_observations(&remote_dirty_usage_acknowledgements); + observation["pending_bucket_count"] = serde_json::json!(pending_dirty_usage_buckets); + let _ = crate::e2e_barrier::checkpoint("ack_send", observation).await; + } let remote_dirty_usage_pending = if remote_dirty_usage_acknowledgements.is_empty() { false } else if let Some(notification_system) = storeapi.scanner_notification_system() { @@ -3949,18 +3969,26 @@ impl ScannerCyclePublicationPlan { /// Re-prove the guardrail after every peer granted a publication lease. The /// comparison uses the guardrail fields (`scanner_activity_guardrail_veto`), /// not the full digest: local generation movement in this window is the next -/// cycle's work and its buckets stay in the dirty map. A peer whose dirty -/// usage moved keeps deferring in this phase, as it does during the walk. +/// cycle's work and its buckets stay in the dirty map. A sealed producer +/// proof may cover peer dirty movement too, but must bind every lease owner +/// and session without changing its candidate or acknowledged tuples. fn scanner_post_lease_activity_defer_reason( baseline: Option<&ScannerActivitySnapshot>, activity: Result, + certification: Option<( + &crate::scanner_io::ScannerRemoteDirtyUsageCertification, + &[crate::storage_api::EcstoreScannerPublicationLeaseGrant], + )>, ) -> Option { match activity { Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) && baseline.is_some_and(|baseline| { scanner_activity_guardrail_veto(baseline, &snapshot).is_none() - && scanner_peer_dirty_usage_moved(baseline, &snapshot).is_none() + && certification.map_or_else( + || scanner_peer_dirty_usage_moved(baseline, &snapshot).is_none(), + |(proof, grants)| proof.matches_publication_leases(grants) && proof.matches_activity(&snapshot), + ) }) => { None @@ -3970,6 +3998,13 @@ fn scanner_post_lease_activity_defer_reason( Err(_) => "probe_failed", Ok(snapshot) if !scanner_activity_allows_usage_publication(snapshot) => "publication_blocked", Ok(_) if baseline.is_none() => "baseline_missing", + Ok(snapshot) + if certification.is_some_and(|(proof, grants)| { + !proof.matches_publication_leases(grants) || !proof.matches_activity(snapshot) + }) => + { + "peer_certification" + } Ok(snapshot) => baseline .and_then(|baseline| { scanner_activity_guardrail_veto(baseline, snapshot) @@ -4133,6 +4168,7 @@ fn finalize_scanner_cycle_result( .is_some_and(|proof| proof.verified_version_for(expected).is_some()) }); let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work() + || scan_cycle_result.has_remote_dirty_usage_pending() || (scan_cycle_result.has_dirty_usage_to_acknowledge() && !durable_complete_snapshot); let remote_dirty_usage_acknowledgements = if durable_complete_snapshot { match proof { diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 3e1a7dc02..b69a08dae 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -546,6 +546,24 @@ pub(crate) struct ScannerNodeActivity { pub(super) publication_blocked: bool, } +#[cfg(feature = "e2e-test-hooks")] +impl ScannerNodeActivity { + pub(crate) fn e2e_observation(&self) -> serde_json::Value { + serde_json::json!({ + "instance_id": self.instance_id, + "namespace_generation": self.namespace_generation, + "maintenance_generation": self.maintenance_generation, + "protocol_version": self.protocol_version, + "topology_digest": crate::e2e_barrier::hex(&self.topology_digest), + "data_movement_active": self.data_movement_active, + "dirty_usage_generation": self.dirty_usage_generation, + "dirty_usage_pending": self.dirty_usage_pending, + "movement_generation": self.movement_generation, + "publication_blocked": self.publication_blocked, + }) + } +} + pub(crate) type ScannerActivitySnapshot = BTreeMap; #[cfg(test)] diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 92eba6042..373994dfe 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -10362,7 +10362,7 @@ fn post_lease_activity_proof_rejects_a_peer_whose_dirty_usage_moved_before_lease (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), ("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3)), ]); - assert_eq!(scanner_post_lease_activity_defer_reason(Some(&before), Ok(before.clone())), None); + assert_eq!(scanner_post_lease_activity_defer_reason(Some(&before), Ok(before.clone()), None), None); // Local generation movement in the lease window is the next cycle's // work: its buckets stay in the dirty map and the walk remains certified. @@ -10373,7 +10373,7 @@ fn post_lease_activity_proof_rejects_a_peer_whose_dirty_usage_moved_before_lease local.namespace_generation += 1; local.dirty_usage_generation += 1; local.dirty_usage_pending = true; - assert_eq!(scanner_post_lease_activity_defer_reason(Some(&before), Ok(local_shaded)), None); + assert_eq!(scanner_post_lease_activity_defer_reason(Some(&before), Ok(local_shaded), None), None); // A peer PUT tail that drained before the lease leaves the movement // generation accepted by the lease unchanged, but the peer's dirty state @@ -10387,14 +10387,14 @@ fn post_lease_activity_proof_rejects_a_peer_whose_dirty_usage_moved_before_lease "the existing movement-only lease remains valid after a PUT tail drains" ); assert!(scanner_activity_allows_usage_publication(&after)); - let reason = scanner_post_lease_activity_defer_reason(Some(&before), Ok(after)); + let reason = scanner_post_lease_activity_defer_reason(Some(&before), Ok(after), None); assert_eq!(reason, Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)); // A guardrail field moving in the window still defers. let mut moved = before.clone(); moved.get_mut("node-2").expect("peer should be present").movement_generation += 1; assert_eq!( - scanner_post_lease_activity_defer_reason(Some(&before), Ok(moved)), + scanner_post_lease_activity_defer_reason(Some(&before), Ok(moved), None), Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) ); @@ -10489,7 +10489,7 @@ fn post_lease_activity_proof_requires_a_complete_matching_baseline() { (Some(&blocked), Ok(blocked.clone())), ] { assert_eq!( - scanner_post_lease_activity_defer_reason(expected, observed), + scanner_post_lease_activity_defer_reason(expected, observed, None), Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) ); } diff --git a/crates/scanner/src/scanner/tests/scoped_ack_publication.rs b/crates/scanner/src/scanner/tests/scoped_ack_publication.rs index ec91fe5c7..71ea32048 100644 --- a/crates/scanner/src/scanner/tests/scoped_ack_publication.rs +++ b/crates/scanner/src/scanner/tests/scoped_ack_publication.rs @@ -49,6 +49,7 @@ async fn complete_candidate(store: &Arc, cycle: u64) -> (crate::scanner service_cohort: None, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ) .await @@ -285,6 +286,7 @@ async fn scoped_ack_publication_observed_candidate_reuse_requires_a_new_root_pro service_cohort: None, resolved_scope_observer: Some(observer), post_walk_gate: None, + peer_observations: None, }, ) .await @@ -456,6 +458,7 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish "remote_scoped_ack_target", "publication_epoch", "remote_lease_targets", + "remote_pending", ] { let (_directory, store) = candidate_store().await; let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await; @@ -502,6 +505,7 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish }, }]), "publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)), + "remote_pending" => scan.with_remote_dirty_usage_pending(true), "remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![( "proof-peer:9000".to_string(), crate::scanner_activity_epoch().to_string(), @@ -524,3 +528,142 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish } crate::scanner_io::clear_dirty_usage_buckets_for_tests(); } + +#[test] +#[serial] +fn certified_empty_remote_ack_keeps_known_remote_work_pending() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let scan = ScannerCycleResult::new(ScannerCycleStatus::Complete, None).with_remote_dirty_usage_pending(true); + let (outcome, pending, acknowledgements) = finalize_scanner_cycle_result(scan, DataUsagePersistOutcome::Current.into()); + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(acknowledgements.is_empty()); + assert!(pending, "an empty ACK cannot hide the remote buckets excluded from certification"); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[test] +fn sealed_peer_proof_binds_every_lease_and_keeps_all_post_lease_guardrails() { + use crate::storage_api::{ + EcstoreScannerPeerDirtyUsageBucket, EcstoreScannerPeerDirtyUsageSnapshot, EcstoreScannerPublicationLease, + EcstoreScannerPublicationLeaseGrant, + }; + let peer_rows = [("peer-a", 101_u128, 201_u128, true), ("peer-b", 102, 202, false)]; + let mut before = BTreeMap::new(); + let mut peers = Vec::new(); + let mut grants = Vec::new(); + for (host, owner, instance, pending) in peer_rows { + let owner_id = uuid::Uuid::from_u128(owner).simple().to_string(); + let instance_id = uuid::Uuid::from_u128(instance).simple().to_string(); + before.insert(host.to_string(), scanner_node_activity_for_tests(&instance_id, 10, 10, pending)); + let buckets = if pending { + BTreeMap::from([( + "cold-bucket".to_string(), + EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: uuid::Uuid::from_u128(301), + generation: 8, + }, + )]) + } else { + BTreeMap::new() + }; + peers.push(( + host.to_string(), + EcstoreScannerPeerDirtyUsageSnapshot { + owner_id: owner_id.clone(), + instance_id: instance_id.clone(), + generation: 10, + pending_bucket_count: buckets.len() as u64, + protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete: true, + buckets, + }, + )); + grants.push(EcstoreScannerPublicationLeaseGrant { + host: host.to_string(), + lease: EcstoreScannerPublicationLease { + token: uuid::Uuid::new_v4(), + movement_generation: 0, + observational_only: false, + owner_id, + session_id: instance_id, + expires_at: std::time::Instant::now() + Duration::from_secs(60), + }, + }); + } + let proof = crate::scanner_io::ScannerRemoteDirtyUsageCertification::for_tests(before.clone(), peers) + .expect("complete producer evidence"); + let mut late_write = before.clone(); + let writer = late_write.get_mut("peer-a").expect("writer"); + writer.namespace_generation += 1; + writer.dirty_usage_generation += 1; + assert_eq!( + scanner_post_lease_activity_defer_reason(Some(&before), Ok(late_write.clone()), Some((&proof, &grants))), + None + ); + + for case in [ + "missing", + "duplicate", + "owner", + "instance", + "movement", + "expired", + "observational", + "token", + ] { + let mut invalid = grants.clone(); + match case { + "missing" => { + invalid.pop(); + } + "duplicate" => invalid[1] = invalid[0].clone(), + "owner" => invalid[1].lease.owner_id = uuid::Uuid::from_u128(999).to_string(), + "instance" => invalid[1].lease.session_id = uuid::Uuid::from_u128(999).to_string(), + "movement" => invalid[1].lease.movement_generation += 1, + "expired" => invalid[1].lease.expires_at = std::time::Instant::now() - Duration::from_secs(1), + "observational" => invalid[1].lease.observational_only = true, + "token" => invalid[1].lease.token = uuid::Uuid::nil(), + _ => unreachable!(), + } + assert_eq!( + scanner_post_lease_activity_defer_reason(Some(&before), Ok(before.clone()), Some((&proof, &invalid))), + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), + "{case}" + ); + } + for field in [ + "instance", + "protocol", + "topology", + "maintenance", + "data_movement", + "movement", + "blocked", + "missing_peer", + "overflow", + "rollback", + ] { + let mut changed = late_write.clone(); + let peer = changed.get_mut("peer-a").expect("peer"); + match field { + "instance" => peer.instance_id = uuid::Uuid::from_u128(999).to_string(), + "protocol" => peer.protocol_version += 1, + "topology" => peer.topology_digest[0] ^= 1, + "maintenance" => peer.maintenance_generation += 1, + "data_movement" => peer.data_movement_active = true, + "movement" => peer.movement_generation += 1, + "blocked" => peer.publication_blocked = true, + "missing_peer" => { + changed.remove("peer-b"); + } + "overflow" => peer.dirty_usage_generation = u64::MAX, + "rollback" => peer.dirty_usage_generation = 9, + _ => unreachable!(), + } + assert_eq!( + scanner_post_lease_activity_defer_reason(Some(&before), Ok(changed), Some((&proof, &grants))), + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), + "{field}" + ); + } +} diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index d9a7b34f9..d22f42e5d 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -84,6 +84,27 @@ impl DataUsagePublicationResult { (self.outcome, self.proof) } + #[cfg(feature = "e2e-test-hooks")] + pub(super) fn e2e_durable_observation( + &self, + epoch: u64, + cycle: u64, + expected: &crate::scanner_io::ScannerPublicationExpectation, + ) -> Option { + let outcome = match self.outcome { + DataUsagePersistOutcome::Saved => "saved", + DataUsagePersistOutcome::AlreadyDurable => "already_durable", + _ => return None, + }; + let (root_version, root_digest) = self.proof.as_ref()?.verified_version_for(expected)?; + let (candidate_digest, coverage_digest) = expected.e2e_digests(); + Some(serde_json::json!({ + "epoch": epoch, "cycle": cycle, "outcome": outcome, + "candidate_sha256": candidate_digest, "coverage_sha256": coverage_digest, + "root_version": root_version, "root_sha256": crate::e2e_barrier::hex(root_digest), + })) + } + pub(super) fn observation_for_attempt( &self, scanner_epoch: u64, @@ -1067,6 +1088,23 @@ where break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded); } + #[cfg(feature = "e2e-test-hooks")] + if !observational + && target_path == DATA_USAGE_OBJ_NAME_PATH.as_str() + && let (Some(expected), Some(epoch), Some(cycle)) = + (ack_expectation.as_ref(), data_usage_info.scanner_epoch, data_usage_info.scanner_cycle) + && expected.matches_encoded_candidate(&data_digest) + { + let (candidate_digest, coverage_digest) = expected.e2e_digests(); + let _ = crate::e2e_barrier::checkpoint( + "root_before_publish", + serde_json::json!({ + "epoch": epoch, "cycle": cycle, + "candidate_sha256": candidate_digest, "coverage_sha256": coverage_digest, + }), + ) + .await; + } let done_save = Metrics::time(Metric::SaveUsage); let (save_result, commit_state) = { let publication_scope = storeapi diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index eeef87851..8bfb7c81f 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -938,6 +938,7 @@ pub struct ScannerBucketScanPlan { publication_epoch: Option, dirty_usage_buckets: Arc, bucket_failures: ScannerBucketFailureState, + bucket_completions: Option>>, pending_maintenance_work: Arc, cache_cycle_floor: Arc, cold_zero_walk_reuse_observed: Arc, @@ -1272,20 +1273,14 @@ enum ScannerCycleActivityStatus { Unverified, } -async fn scanner_cycle_activity_status( - store: &S, - distributed: bool, +fn scanner_cycle_activity_status( before: &crate::scanner::ScannerActivitySnapshot, -) -> (ScannerCycleActivityStatus, Vec<(String, String, u64)>) -where - S: ScannerStorage, -{ - // Read the storage movement barrier before sampling the activity - // generations. A movement that starts during this await must invalidate - // the earlier baseline. Ordinary PUT tails are not part of the barrier: - // they shade their bucket and are certified per bucket below. - let publication_blocked = store.scanner_data_usage_publication_blocked().await; - match crate::scanner::probe_scanner_activity(store, distributed).await { + observed: std::result::Result, + publication_blocked: bool, + certification: Option<&ScannerRemoteDirtyUsageCertification>, + certification_required: bool, +) -> (ScannerCycleActivityStatus, Vec<(String, String, u64)>) { + match observed { Ok(after) => { let veto = if publication_blocked { Some(crate::scanner::ScannerActivityGuardrailVeto::new( @@ -1309,10 +1304,15 @@ where ); global_metrics().record_scanner_guardrail_veto(veto.reason()); ScannerCycleActivityStatus::Changed - } else if let Some(moved) = crate::scanner::scanner_peer_dirty_usage_moved(before, &after) { - // Per-bucket certification covers the local dirty map only; - // a peer whose dirty state moved keeps today's whole-cycle - // outcome until scoped acknowledgements can name its buckets. + } else if certification_required && certification.is_none() + || certification.is_some_and(|proof| !proof.matches_activity(&after)) + { + ScannerCycleActivityStatus::Unverified + } else if let Some(moved) = + crate::scanner::scanner_peer_dirty_usage_moved(before, &after).filter(|_| certification.is_none()) + { + // Without a complete producer proof, retain the original + // whole-cycle fallback for a moving peer. tracing::info!( target: "rustfs::scanner::io", event = EVENT_SCANNER_SET_STATE, @@ -1743,6 +1743,8 @@ pub(crate) struct ScannerCycleResult { partial_bucket_progress: bool, dirty_usage_clear: Option, remote_dirty_usage_acknowledgements: Vec, + remote_dirty_usage_certification: Option, + remote_dirty_usage_pending: bool, distributed_segment_invalidation_evidence: Option, segment_reuse_activation_preflight: ScannerSegmentReuseActivationPreflight, remote_publication_lease_targets: Vec<(String, String, u64)>, @@ -1765,6 +1767,8 @@ impl ScannerCycleResult { partial_bucket_progress: false, dirty_usage_clear, remote_dirty_usage_acknowledgements: Vec::new(), + remote_dirty_usage_certification: None, + remote_dirty_usage_pending: false, distributed_segment_invalidation_evidence: None, segment_reuse_activation_preflight: scanner_segment_reuse_activation_preflight(), remote_publication_lease_targets: Vec::new(), @@ -1888,6 +1892,26 @@ impl ScannerCycleResult { self } + fn with_remote_dirty_usage_certification(mut self, certification: Option) -> Self { + self.publication_expectation = None; + self.remote_dirty_usage_certification = certification; + self + } + + pub(crate) fn remote_dirty_usage_certification(&self) -> Option<&ScannerRemoteDirtyUsageCertification> { + self.remote_dirty_usage_certification.as_ref() + } + + pub(crate) fn with_remote_dirty_usage_pending(mut self, pending: bool) -> Self { + self.publication_expectation = None; + self.remote_dirty_usage_pending = pending; + self + } + + pub(crate) fn has_remote_dirty_usage_pending(&self) -> bool { + self.remote_dirty_usage_pending + } + fn with_distributed_segment_invalidation_evidence( mut self, evidence: Option, @@ -1969,9 +1993,12 @@ mod guards; pub(crate) use guards::ScannerServiceCohort; mod io_cache; mod io_cycle; +mod peer_certification; #[cfg(test)] use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan}; pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped}; +pub(crate) use peer_certification::ScannerRemoteDirtyUsageCertification; +use peer_certification::{ScannerBucketCompletions, ScannerRemoteDirtyUsageWindow}; mod io_disk; #[cfg(test)] mod publish_gate_tests; diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index e7cd42bbb..30b792bf2 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -589,6 +589,11 @@ impl ScannerPublicationExpectation { Arc::ptr_eq(&self.candidate, &other.candidate) && self.candidate.1 == other.candidate.1 } + #[cfg(feature = "e2e-test-hooks")] + pub(crate) fn e2e_digests(&self) -> (String, String) { + (crate::e2e_barrier::hex(&self.candidate.0), crate::e2e_barrier::hex(&self.candidate.1.0)) + } + #[cfg(test)] pub(crate) fn for_tests(candidate_digest: [u8; 32], coverage_digest: DataUsageScanPlanDigest) -> Self { Self { diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 670f93887..5882d705c 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -282,6 +282,7 @@ impl ScannerIOCache for SetDisks { publication_epoch, dirty_usage_buckets, bucket_failures, + bucket_completions, pending_maintenance_work, cache_cycle_floor, cold_zero_walk_reuse_observed, @@ -1296,6 +1297,12 @@ impl ScannerIOCache for SetDisks { let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::(1); let scan_ctx = ctx_clone.child_token(); + #[cfg(feature = "e2e-test-hooks")] + let _ = crate::e2e_barrier::checkpoint("bucket_walk", serde_json::json!({ + "epoch": leader_epoch, "cycle": want_cycle, + "pool": source.pool_index, "set": source.set_index, + "bucket": bucket.name, "drive": disk.path().display().to_string(), + })).await; let scan = disk.clone().nsscanner_disk( scan_ctx.clone(), budget_clone.clone(), @@ -1707,6 +1714,22 @@ impl ScannerIOCache for SetDisks { return Err(err); } + // Record only actual worker completions, before adding carried values. + // The root's cold entries cannot certify a newly observed peer tuple. + if let Some(completions) = bucket_completions { + let completed_incarnations = completed_buckets + .iter() + .filter_map(|bucket| { + current_bucket_incarnations + .as_ref()? + .get(bucket) + .filter(|incarnation| !incarnation.is_nil()) + .map(|incarnation| (bucket.clone(), *incarnation)) + }) + .collect(); + completions.lock().await.insert(source, completed_incarnations); + } + let carried_buckets = carry_partial_bucket_values( &mut cache, &old_cache, diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 938a77573..40a9adc6b 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -79,6 +79,8 @@ where resolved_scope_observer: None, #[cfg(test)] post_walk_gate: None, + #[cfg(test)] + peer_observations: None, }; nsscanner_with_storage_status_scoped(store, request).await } @@ -103,10 +105,133 @@ pub(crate) struct ScannerCycleRequest { /// certification regardless of how fast the walk finishes. #[cfg(test)] pub(crate) post_walk_gate: Option>, + #[cfg(test)] + pub(crate) peer_observations: Option, +} + +#[cfg(test)] +#[derive(Clone)] +pub(crate) struct ScannerCyclePeerObservations { + pub(crate) activity: Arc crate::scanner::ScannerActivitySnapshot + Send + Sync>, + pub(crate) dirty_usage: Arc Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)> + Send + Sync>, + pub(crate) scoped_ack_capable: bool, +} + +pub(super) async fn probe_scanner_cycle_activity( + store: &S, + distributed: bool, + #[cfg(test)] peer_observations: Option<&ScannerCyclePeerObservations>, +) -> std::result::Result +where + S: ScannerStorage, +{ + #[cfg(test)] + if let Some(peers) = peer_observations { + let mut snapshot = crate::scanner::probe_scanner_activity(store, false).await?; + for (host, activity) in (peers.activity)() { + if snapshot.insert(host, activity).is_some() { + return Err("duplicate test peer activity".to_string()); + } + } + return Ok(snapshot); + } + crate::scanner::probe_scanner_activity(store, distributed).await +} + +async fn observe_remote_dirty_usage_window( + store: &S, + before: &crate::scanner::ScannerActivitySnapshot, + _stage: &str, + _epoch: u64, + _cycle: u64, + #[cfg(test)] peer_observations: Option<&ScannerCyclePeerObservations>, +) -> ( + Option, + std::result::Result, +) +where + S: ScannerStorage, +{ + #[cfg(feature = "e2e-test-hooks")] + let _ = + crate::e2e_barrier::checkpoint(&format!("before_{_stage}"), serde_json::json!({"epoch": _epoch, "cycle": _cycle})).await; + #[cfg(test)] + let snapshots = if let Some(peers) = peer_observations { + Ok((peers.dirty_usage)()) + } else if let Some(notification) = store.scanner_notification_system() { + notification + .scanner_dirty_usage_snapshots() + .await + .map_err(|error| error.to_string()) + } else { + Err("scanner peer snapshot service is unavailable".to_string()) + }; + #[cfg(not(test))] + let snapshots = if let Some(notification) = store.scanner_notification_system() { + notification + .scanner_dirty_usage_snapshots() + .await + .map_err(|error| error.to_string()) + } else { + Err("scanner peer snapshot service is unavailable".to_string()) + }; + let after = probe_scanner_cycle_activity( + store, + true, + #[cfg(test)] + peer_observations, + ) + .await; + let window = match (snapshots, &after) { + (Ok(snapshots), Ok(after)) => ScannerRemoteDirtyUsageWindow::verify(before.clone(), snapshots, after.clone()), + _ => None, + }; + #[cfg(feature = "e2e-test-hooks")] + let _ = crate::e2e_barrier::checkpoint( + _stage, + window.as_ref().map_or_else( + || serde_json::json!({"epoch": _epoch, "cycle": _cycle, "verified": false}), + |window| window.observation(_epoch, _cycle), + ), + ) + .await; + (window, after) +} + +async fn remote_dirty_usage_window_supports_scoped_ack( + store: &S, + window: &ScannerRemoteDirtyUsageWindow, + #[cfg(test)] peer_observations: Option<&ScannerCyclePeerObservations>, +) -> bool +where + S: ScannerStorage, +{ + let acknowledgements = window.acknowledgements(); + if scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&acknowledgements) { + return false; + } + // Empty peers remain in the proof; they need no empty capability request. + if acknowledgements.is_empty() { + return true; + } + #[cfg(test)] + if let Some(peers) = peer_observations { + return peers.scoped_ack_capable; + } + let Some(notification) = store.scanner_notification_system() else { + return false; + }; + matches!( + notification + .scanner_scoped_dirty_usage_capabilities(acknowledgements.into_iter().map(Into::into).collect(),) + .await, + Ok(true) + ) } pub(super) struct ScannerBucketScopeResolution<'a> { pub(super) requested_scope: ScannerBucketScanScope, + pub(super) remote_window: Option<&'a ScannerRemoteDirtyUsageWindow>, pub(super) baseline_proof: ScannerCacheBaselineProof<'a>, pub(super) activity_before: &'a crate::scanner::ScannerActivitySnapshot, pub(super) dirty_usage_snapshot: &'a DirtyUsageSnapshot, @@ -149,6 +274,29 @@ where .cloned() .collect::>(); if distributed { + if let Some(window) = resolution.remote_window { + if let Some(strict_remote_usage) = window.strict_remote_usage() { + // Only the original quiet window can supply segment reuse evidence. + return resolve_remote_dirty_usage_scope( + resolution.requested_scope, + resolution.dirty_usage_snapshot, + strict_remote_usage, + resolution.all_buckets, + resolution.baseline_proof, + ); + } + let mut dirty_buckets = dirty_buckets; + dirty_buckets.extend(window.dirty_buckets()); + return default_result(scoped_scan_scope_from_dirty_buckets( + resolution.requested_scope, + dirty_buckets, + None, + true, + false, + resolution.all_buckets, + resolution.baseline_proof, + )); + } let notification_system = store.scanner_notification_system(); #[cfg(test)] let peer_snapshots = if let Some(peer_snapshots) = resolution.test_peer_snapshots.clone() { @@ -322,7 +470,11 @@ where resolved_scope_observer, #[cfg(test)] post_walk_gate, + #[cfg(test)] + peer_observations, } = request; + #[cfg(feature = "e2e-test-hooks")] + let requested_scope = e2e_scope_observation(&scan_scope); let child_token = ctx.child_token(); let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); @@ -359,7 +511,17 @@ where }; let distributed = store.setup_is_dist_erasure().await; - let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(store, distributed).await) { + #[cfg(test)] + let distributed = distributed || peer_observations.is_some(); + let activity_before = match scanner_activity_preflight( + probe_scanner_cycle_activity( + store, + distributed, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await, + ) { ScannerActivityPreflight::Ready(snapshot) => Arc::new(snapshot), ScannerActivityPreflight::ActivityBaselineUnavailable(err) => { warn!( @@ -420,6 +582,39 @@ where .unwrap_or_else(|poisoned| poisoned.into_inner()) .refresh(&buckets_by_source); } + let full_scan_required = requires_full_scan || scan_mode == HealScanMode::Deep; + let scope_eligible = !full_scan_required && scan_scope.is_default(); + let mut remote_start = if distributed && scope_eligible && !all_buckets.is_empty() { + observe_remote_dirty_usage_window( + store, + &activity_before, + "start_snapshot", + leader_epoch, + want_cycle, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await + .0 + } else { + None + }; + if let Some(window) = remote_start.as_ref() + && !remote_dirty_usage_window_supports_scoped_ack( + store, + window, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await + { + remote_start = None; + } + // Work and Current-cache identities include writes admitted by the start + // window. Its earlier edge remains in the producer proof, not the cache key. + let activity_before = remote_start + .as_ref() + .map_or(activity_before, ScannerRemoteDirtyUsageWindow::activity_for_walk); let structural_scan_plan_digest = scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before)); let scan_plan_digest = scanner_bucket_work_digest(structural_scan_plan_digest, scan_mode, requires_full_scan); @@ -437,22 +632,21 @@ where want_cycle, scan_plan_digest: structural_scan_plan_digest, }; - let full_scan_required = requires_full_scan || scan_mode == HealScanMode::Deep; - let scope_eligible = !full_scan_required && scan_scope.is_default(); let scope_resolution = resolve_scanner_bucket_scan_scope( store, distributed, ScannerBucketScopeResolution { requested_scope: scan_scope, + remote_window: remote_start.as_ref(), baseline_proof, activity_before: &activity_before, dirty_usage_snapshot: &dirty_usage_snapshot, all_buckets: &all_buckets, requires_full_scan: full_scan_required, #[cfg(test)] - test_peer_snapshots: None, + test_peer_snapshots: peer_observations.as_ref().map(|peers| (peers.dirty_usage)()), #[cfg(test)] - test_scoped_dirty_usage_capability: None, + test_scoped_dirty_usage_capability: peer_observations.as_ref().map(|peers| peers.scoped_ack_capable), }, ) .await; @@ -496,8 +690,16 @@ where if !bucket_plan_complete { return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)); } + let publication_blocked = store.scanner_data_usage_publication_blocked().await; + let activity_after = probe_scanner_cycle_activity( + store, + distributed, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await; let (activity_status, remote_publication_lease_targets) = - scanner_cycle_activity_status(store, distributed, &activity_before).await; + scanner_cycle_activity_status(&activity_before, activity_after, publication_blocked, None, false); let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); let status = classify_nsscanner_cycle( true, @@ -583,6 +785,15 @@ where let serial_progress_accounting = budget.requires_serial_progress_accounting(); let set_scan_limit = scanner_budgeted_concurrency_limit(dispatch_limit, serial_progress_accounting); let bucket_failures = ScannerBucketFailureState::default(); + let bucket_completions = remote_start + .as_ref() + .map(|_| Arc::new(Mutex::new(ScannerBucketCompletions::new()))); + // Recovery sweeps have no remote certification window, but test sessions + // still need the actual worker completions before any values are carried. + #[cfg(feature = "e2e-test-hooks")] + let bucket_completions = bucket_completions.or_else(|| { + std::env::var_os("RUSTFS_E2E_SCANNER_BARRIER_DIR").map(|_| Arc::new(Mutex::new(ScannerBucketCompletions::new()))) + }); let pending_maintenance_work = Arc::new(AtomicBool::new(false)); let cold_zero_walk_reuse_observed = Arc::new(AtomicBool::new(false)); record_set_scan_concurrency_limit(set_scan_limit); @@ -679,6 +890,7 @@ where publication_epoch, dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), bucket_failures: bucket_failures.clone(), + bucket_completions: bucket_completions.clone(), pending_maintenance_work: pending_maintenance_work.clone(), cache_cycle_floor: cache_cycle_floor.clone(), cold_zero_walk_reuse_observed: cold_zero_walk_reuse_observed.clone(), @@ -827,8 +1039,43 @@ where !namespace_not_found_buckets.is_empty(), ); let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); - let (activity_status, remote_publication_lease_targets) = - scanner_cycle_activity_status(store, distributed, &activity_before).await; + // Check the movement barrier before sampling either edge. The two RPC + // windows are single observations; publication never waits for silence. + let publication_blocked = store.scanner_data_usage_publication_blocked().await; + let post_activity_before = probe_scanner_cycle_activity( + store, + distributed, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await; + let remote_certification_attempted = remote_start.is_some(); + let (remote_certification, post_activity) = match (remote_start, post_activity_before) { + (Some(start), Ok(before)) => { + let (post_walk, after) = observe_remote_dirty_usage_window( + store, + &before, + "post_walk_snapshot", + leader_epoch, + want_cycle, + #[cfg(test)] + peer_observations.as_ref(), + ) + .await; + ( + post_walk.and_then(|post_walk| ScannerRemoteDirtyUsageCertification::new(start, post_walk)), + after, + ) + } + (_, after) => (None, after), + }; + let (activity_status, remote_publication_lease_targets) = scanner_cycle_activity_status( + &activity_before, + post_activity, + publication_blocked, + remote_certification.as_ref(), + remote_certification_attempted, + ); let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::>(); let completed_usage = completed_usage_candidate( &results, @@ -877,17 +1124,93 @@ where // stopped early is pending for the same reason: the publication carries // its last certified value, not this walk's (a `Complete` cycle has only // carried partial buckets, see `classify_nsscanner_cycle`). + let mut uncertified_buckets = failed_buckets.union(&partial_buckets).cloned().collect::>(); + let local_pending_buckets = pending_dirty_usage_buckets(&dirty_usage_snapshot.buckets); + uncertified_buckets.extend(local_pending_buckets.iter().cloned()); + let completions = match bucket_completions { + Some(completions) => completions.lock().await.clone(), + None => ScannerBucketCompletions::new(), + }; + let (certified_acknowledgements, remote_pending_buckets) = remote_certification.as_ref().map_or_else( + || (Vec::new(), ScannerCyclePendingBuckets::new()), + |proof| proof.classify_buckets(&expected_sources, &completions, &uncertified_buckets), + ); + let remote_dirty_usage_pending = !remote_pending_buckets.is_empty(); + uncertified_buckets.extend(remote_pending_buckets.iter().cloned()); let pending_buckets = if cycle_status == ScannerCycleStatus::Complete { - let mut pending = pending_dirty_usage_buckets(&dirty_usage_snapshot.buckets); + let mut pending = local_pending_buckets; pending.extend(partial_buckets.iter().cloned()); + pending.extend(remote_pending_buckets); pending } else { ScannerCyclePendingBuckets::new() }; + // A verified but empty scoped ACK is deliberate. It must never become a + // whole-generation ACK that could clear a hot or carried partial bucket. + let remote_dirty_usage_acknowledgements = if cycle_status != ScannerCycleStatus::Complete { + Vec::new() + } else if remote_certification.is_some() { + certified_acknowledgements + } else if remote_certification_attempted { + Vec::new() + } else if !remote_dirty_usage_acknowledgements.is_empty() { + remote_dirty_usage_acknowledgements + } else if scan_scope.is_default() { + crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before) + } else { + Vec::new() + }; let mut publication_expectation = None; let usage_update_emitted = if let Some(candidate) = completed_usage { let (data_usage_info, expectation) = candidate.prepare(cycle_status, &pending_buckets); publication_expectation = expectation; + #[cfg(feature = "e2e-test-hooks")] + { + let digests = publication_expectation + .as_ref() + .map(ScannerPublicationExpectation::e2e_digests); + let scanned_buckets = completions + .values() + .flat_map(|buckets| buckets.keys().cloned()) + .collect::>(); + let mut set_coverage = completions.iter().collect::>(); + set_coverage.sort_by_key(|(source, _)| (source.pool_index, source.set_index)); + let _ = crate::e2e_barrier::checkpoint( + "cycle_candidate", + serde_json::json!({ + "epoch": leader_epoch, + "cycle": want_cycle, + "scan_mode": if scan_mode == HealScanMode::Deep { "deep" } else { "normal" }, + "requested_scope": requested_scope, + "selected_scope": e2e_scope_observation(&scan_scope), + "scanned_buckets": scanned_buckets, + "pending_buckets": data_usage_info.usage_snapshot_pending_buckets, + "converged": data_usage_info.usage_snapshot_converged, + "status": match cycle_status { + ScannerCycleStatus::Complete => "complete", + ScannerCycleStatus::Incomplete => "incomplete", + ScannerCycleStatus::Superseded => "superseded", + ScannerCycleStatus::Deferred(_) => "deferred", + }, + "candidate_sha256": digests.as_ref().map(|digests| &digests.0), + "coverage_sha256": digests.as_ref().map(|digests| &digests.1), + "peer_certification": remote_certification.is_some(), + "certified_peers": remote_certification.as_ref().map(|proof| { + proof.post_walk.peers.iter().map(|(host, peer)| serde_json::json!({ + "host": host, "owner_id": peer.owner_id, "instance_id": peer.instance_id, + })).collect::>() + }).unwrap_or_default(), + "set_coverage": set_coverage.into_iter().map(|(source, buckets)| serde_json::json!({ + "pool": source.pool_index, + "set": source.set_index, + "buckets": buckets.keys().cloned().collect::>(), + "bucket_incarnations": buckets, + })).collect::>(), + "acks": crate::e2e_barrier::acknowledgement_observations(&remote_dirty_usage_acknowledgements), + }), + ) + .await; + } if should_publish_observational_snapshot(cycle_status) { publish_observational_snapshot(&updates, data_usage_info).await? } else { @@ -902,7 +1225,6 @@ where }; // A walk that stopped early certifies nothing: the bucket keeps its // dirty generation like a failed one and is walked again next cycle. - let uncertified_buckets = failed_buckets.union(&partial_buckets).cloned().collect::>(); let dirty_usage_clear = should_clear_dirty_usage_snapshot( result.is_ok(), structurally_complete_snapshot, @@ -915,14 +1237,6 @@ where if cycle_status == ScannerCycleStatus::Complete { complete_tier_registry_cycle(want_cycle, leader_epoch); } - let remote_dirty_usage_acknowledgements = - if cycle_status == ScannerCycleStatus::Complete && !remote_dirty_usage_acknowledgements.is_empty() { - remote_dirty_usage_acknowledgements - } else if cycle_status == ScannerCycleStatus::Complete && scan_scope.is_default() { - crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before) - } else { - Vec::new() - }; Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) .with_publication_epoch(publication_epoch) .with_activity_digest(activity_digest) @@ -932,6 +1246,8 @@ where .with_partial_bucket_progress(partial_bucket_progress) .with_remote_publication_lease_targets(remote_publication_lease_targets) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) + .with_remote_dirty_usage_certification(remote_certification) + .with_remote_dirty_usage_pending(remote_dirty_usage_pending) .with_distributed_segment_invalidation_evidence(distributed_segment_invalidation_evidence) .with_segment_reuse_activation_preflight(segment_reuse_activation_preflight) .with_failed_dirty_usage(!failed_buckets.is_empty()) @@ -940,3 +1256,11 @@ where .with_scope_fallback(scope_fallback) .with_publication_expectation(publication_expectation)) } + +#[cfg(feature = "e2e-test-hooks")] +fn e2e_scope_observation(scope: &ScannerBucketScanScope) -> serde_json::Value { + match &scope.selected_buckets { + Some(buckets) => serde_json::json!({"kind": "buckets", "buckets": buckets.iter().collect::>()}), + None => serde_json::json!({"kind": "all"}), + } +} diff --git a/crates/scanner/src/scanner_io/peer_certification.rs b/crates/scanner/src/scanner_io/peer_certification.rs new file mode 100644 index 000000000..dd88f0595 --- /dev/null +++ b/crates/scanner/src/scanner_io/peer_certification.rs @@ -0,0 +1,582 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::scanner::{ + ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, ScannerDirtyUsageAcknowledgementKind, + scanner_activity_allows_usage_publication, scanner_activity_dirty_usage_state_for_host, scanner_activity_guardrail_veto, + scanner_activity_publication_lease_targets, +}; +use crate::storage_api::EcstoreScannerPublicationLeaseGrant; + +/// Only worker results received in this cycle enter this map. A carried value +/// or a cold bucket copied from the old set root is not a completion. +pub(super) type ScannerBucketCompletions = HashMap>; + +#[derive(Debug)] +pub(super) struct ScannerRemoteDirtyUsageWindow { + pub(super) before: ScannerActivitySnapshot, + pub(super) after: ScannerActivitySnapshot, + pub(super) peers: BTreeMap, +} + +fn remote_generations_advance(before: &ScannerActivitySnapshot, after: &ScannerActivitySnapshot) -> bool { + scanner_activity_allows_usage_publication(before) + && scanner_activity_allows_usage_publication(after) + && scanner_activity_guardrail_veto(before, after).is_none() + && scanner_activity_publication_lease_targets(before).iter().all(|(host, _, _)| { + let Some((_, before_generation, _)) = scanner_activity_dirty_usage_state_for_host(before, host) else { + return false; + }; + scanner_activity_dirty_usage_state_for_host(after, host) + .is_some_and(|(_, after_generation, _)| before_generation <= after_generation && after_generation < u64::MAX) + }) +} + +impl ScannerRemoteDirtyUsageWindow { + /// The caller obtains these in order: before activity, dirty snapshot, + /// after activity. The generation can advance without waiting for silence. + pub(super) fn verify( + before: ScannerActivitySnapshot, + snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>, + after: ScannerActivitySnapshot, + ) -> Option { + let expected = scanner_activity_publication_lease_targets(&before); + if expected.is_empty() || expected.len() != snapshots.len() || !remote_generations_advance(&before, &after) { + return None; + } + let mut peers = BTreeMap::new(); + for (host, snapshot) in snapshots { + let (instance_id, lower, lower_pending) = scanner_activity_dirty_usage_state_for_host(&before, &host)?; + let (_, upper, upper_pending) = scanner_activity_dirty_usage_state_for_host(&after, &host)?; + let pending = !snapshot.buckets.is_empty(); + if host.is_empty() + || snapshot.instance_id != instance_id + || uuid::Uuid::parse_str(&snapshot.owner_id) + .ok() + .is_none_or(|owner| owner.is_nil()) + || uuid::Uuid::parse_str(&snapshot.instance_id) + .ok() + .is_none_or(|instance| instance.is_nil()) + || snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + || !snapshot.complete + || snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + || snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).ok()? + || !(lower <= snapshot.generation && snapshot.generation <= upper && upper < u64::MAX) + || (lower == snapshot.generation && lower_pending != pending) + || (upper == snapshot.generation && upper_pending != pending) + || snapshot.buckets.iter().any(|(bucket, entry)| { + bucket.is_empty() + || bucket == DATA_USAGE_ROOT + || entry.bucket_incarnation.is_nil() + || entry.generation == 0 + || entry.generation > snapshot.generation + }) + || peers.insert(host, snapshot).is_some() + { + return None; + } + } + (peers.len() == expected.len() && expected.iter().all(|(host, _, _)| peers.contains_key(host))).then_some(Self { + before, + after, + peers, + }) + } + + pub(super) fn activity_for_walk(&self) -> Arc { + Arc::new(self.after.clone()) + } + + pub(super) fn strict_remote_usage(&self) -> Option { + if crate::scanner::scanner_peer_dirty_usage_moved(&self.before, &self.after).is_some() { + return None; + } + let expected = self + .peers + .keys() + .map(|host| { + let (instance_id, generation, pending) = scanner_activity_dirty_usage_state_for_host(&self.before, host)?; + Some(( + host.clone(), + ScannerPeerDirtyUsageExpectation { + instance_id: instance_id.to_string(), + generation, + pending, + }, + )) + }) + .collect::>>()?; + verified_remote_dirty_usage(&expected, self.peers.iter().map(|(host, peer)| (host.clone(), peer.clone())).collect()) + } + + #[cfg(feature = "e2e-test-hooks")] + pub(super) fn observation(&self, epoch: u64, cycle: u64) -> serde_json::Value { + serde_json::json!({ + "epoch": epoch, + "cycle": cycle, + "verified": true, + "activity_before": crate::e2e_barrier::activity_observations(&self.before), + "activity_after": crate::e2e_barrier::activity_observations(&self.after), + "peers": self.peers.iter().map(|(host, peer)| serde_json::json!({ + "host": host, + "owner_id": peer.owner_id, + "instance_id": peer.instance_id, + "generation": peer.generation, + "protocol_version": peer.protocol_version, + "complete": peer.complete, + "pending_bucket_count": peer.pending_bucket_count, + "buckets": peer.buckets.iter().map(|(bucket, entry)| serde_json::json!({ + "bucket": bucket, + "bucket_incarnation": entry.bucket_incarnation, + "generation": entry.generation, + })).collect::>(), + })).collect::>(), + }) + } + + pub(super) fn dirty_buckets(&self) -> HashSet { + self.peers.values().flat_map(|peer| peer.buckets.keys().cloned()).collect() + } + + pub(super) fn acknowledgements(&self) -> Vec { + self.peers + .iter() + .filter(|(_, peer)| !peer.buckets.is_empty()) + .map(|(host, peer)| ScannerDirtyUsageAcknowledgement { + host: host.clone(), + instance_id: peer.instance_id.clone(), + kind: ScannerDirtyUsageAcknowledgementKind::Scoped { + owner_id: peer.owner_id.clone(), + entries: peer + .buckets + .iter() + .map(|(bucket, entry)| crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry { + bucket: bucket.clone(), + bucket_incarnation: entry.bucket_incarnation, + generation: entry.generation, + }) + .collect(), + }, + }) + .collect() + } +} + +/// A complete producer proof includes clean peers even when it yields no ACK. +/// The start and post-walk maps remain sealed until publication and lease +/// validation; post-lease writes are retained by exact receiver comparison. +#[derive(Debug)] +pub(crate) struct ScannerRemoteDirtyUsageCertification { + pub(super) start: ScannerRemoteDirtyUsageWindow, + pub(super) post_walk: ScannerRemoteDirtyUsageWindow, +} + +impl ScannerRemoteDirtyUsageCertification { + pub(super) fn new(start: ScannerRemoteDirtyUsageWindow, post_walk: ScannerRemoteDirtyUsageWindow) -> Option { + if !remote_generations_advance(&start.after, &post_walk.before) + || start.peers.len() != post_walk.peers.len() + || start.peers.iter().any(|(host, before)| { + post_walk.peers.get(host).is_none_or(|after| { + before.owner_id != after.owner_id + || before.instance_id != after.instance_id + || before.protocol_version != after.protocol_version + || before.generation > after.generation + }) + }) + { + return None; + } + Some(Self { start, post_walk }) + } + + #[cfg(test)] + pub(crate) fn for_tests( + activity: ScannerActivitySnapshot, + peers: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>, + ) -> Option { + let start = ScannerRemoteDirtyUsageWindow::verify(activity.clone(), peers.clone(), activity.clone())?; + let post = ScannerRemoteDirtyUsageWindow::verify(activity.clone(), peers, activity)?; + Self::new(start, post) + } + + pub(super) fn classify_buckets( + &self, + expected_sources: &HashSet, + completions: &ScannerBucketCompletions, + uncertified: &HashSet, + ) -> (Vec, ScannerCyclePendingBuckets) { + let mut buckets = self.start.dirty_buckets(); + buckets.extend(self.post_walk.dirty_buckets()); + let sources_complete = !expected_sources.is_empty() + && completions.len() == expected_sources.len() + && expected_sources.iter().all(|source| completions.contains_key(source)); + let pending = buckets + .iter() + .filter(|bucket| { + !sources_complete + || uncertified.contains(*bucket) + || self.start.peers.iter().any(|(host, before)| { + let start = before.buckets.get(*bucket); + let post_walk = self.post_walk.peers[host].buckets.get(*bucket); + start != post_walk + || start.is_some_and(|entry| { + expected_sources.iter().any(|source| { + completions.get(source).and_then(|set| set.get(*bucket)) != Some(&entry.bucket_incarnation) + }) + }) + }) + }) + .cloned() + .collect::(); + let acknowledgements = self + .start + .acknowledgements() + .into_iter() + .filter_map(|mut acknowledgement| { + let ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } = &mut acknowledgement.kind else { + return None; + }; + entries.retain(|entry| !pending.contains(&entry.bucket)); + (!entries.is_empty()).then_some(acknowledgement) + }) + .collect(); + (acknowledgements, pending) + } + + pub(crate) fn matches_activity(&self, activity: &ScannerActivitySnapshot) -> bool { + remote_generations_advance(&self.post_walk.after, activity) + } + + pub(crate) fn matches_publication_leases(&self, grants: &[EcstoreScannerPublicationLeaseGrant]) -> bool { + if grants.len() != self.post_walk.peers.len() { + return false; + } + let targets = scanner_activity_publication_lease_targets(&self.post_walk.after) + .into_iter() + .map(|(host, instance, movement)| (host, (instance, movement))) + .collect::>(); + let mut hosts = HashSet::new(); + grants.iter().all(|grant| { + let Some(peer) = self.post_walk.peers.get(&grant.host) else { + return false; + }; + hosts.insert(grant.host.as_str()) + && grant.lease.is_valid() + && !grant.lease.observational_only + && !grant.lease.token.is_nil() + && grant.lease.owner_id == peer.owner_id + && grant.lease.session_id == peer.instance_id + && targets.get(&grant.host).is_some_and(|(instance, movement)| { + instance == &grant.lease.session_id && *movement == grant.lease.movement_generation + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HOT: uuid::Uuid = uuid::Uuid::from_u128(71); + const COLD: uuid::Uuid = uuid::Uuid::from_u128(72); + + fn peers(generation: u64, hot_generation: u64) -> Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)> { + ["peer-a", "peer-b"] + .into_iter() + .enumerate() + .map(|(index, host)| { + let buckets = if index == 0 { + BTreeMap::from([ + ( + "cold-bucket".to_string(), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: COLD, + generation: 8, + }, + ), + ( + "hot-bucket".to_string(), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: HOT, + generation: hot_generation, + }, + ), + ]) + } else { + BTreeMap::new() + }; + ( + host.to_string(), + EcstoreScannerPeerDirtyUsageSnapshot { + owner_id: uuid::Uuid::from_u128(101 + index as u128).simple().to_string(), + instance_id: uuid::Uuid::from_u128(201 + index as u128).simple().to_string(), + generation: if index == 0 { generation } else { 7 }, + pending_bucket_count: buckets.len() as u64, + protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, + complete: true, + buckets, + }, + ) + }) + .collect() + } + + fn activity(generation: u64) -> ScannerActivitySnapshot { + peers(generation, generation) + .into_iter() + .map(|(host, peer)| { + ( + host, + crate::scanner::scanner_node_activity_for_tests( + &peer.instance_id, + peer.generation, + peer.generation, + !peer.buckets.is_empty(), + ), + ) + }) + .collect() + } + + fn window(lower: u64, generation: u64, upper: u64) -> ScannerRemoteDirtyUsageWindow { + ScannerRemoteDirtyUsageWindow::verify(activity(lower), peers(generation, generation), activity(upper)) + .expect("ordered complete producer window") + } + + fn completions() -> (HashSet, ScannerBucketCompletions) { + let sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(0, 1)]); + let completed = sources + .iter() + .map(|source| { + ( + *source, + HashMap::from([("cold-bucket".to_string(), COLD), ("hot-bucket".to_string(), HOT)]), + ) + }) + .collect(); + (sources, completed) + } + + #[test] + fn hot_writes_cross_both_bounded_windows_while_the_clean_peer_stays_in_the_proof() { + let start = window(9, 10, 11); + assert!(start.strict_remote_usage().is_none(), "the relaxed window grants no segment reuse proof"); + let proof = ScannerRemoteDirtyUsageCertification::new(start, window(12, 13, 14)).expect("same producers"); + let (sources, completed) = completions(); + let (acks, pending) = proof.classify_buckets(&sources, &completed, &HashSet::new()); + assert_eq!(pending, BTreeSet::from(["hot-bucket".to_string()])); + assert_eq!(acks.len(), 1); + assert_eq!(proof.post_walk.peers.len(), 2); + assert!(proof.post_walk.peers["peer-b"].buckets.is_empty()); + let ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } = &acks[0].kind else { + panic!("scoped ACK") + }; + assert_eq!(entries.len(), 1); + assert_eq!( + (entries[0].bucket.as_str(), entries[0].bucket_incarnation, entries[0].generation), + ("cold-bucket", COLD, 8) + ); + } + + #[test] + fn producer_windows_reject_missing_duplicate_stale_future_overflow_and_incomplete_evidence() { + for case in [ + "missing_peer", + "duplicate_peer", + "old_snapshot", + "future_snapshot", + "overflow", + "owner", + "instance", + "protocol", + "incomplete", + "count", + "future_entry", + "nil_incarnation", + "empty_bucket", + "entry_limit", + "pending", + ] { + let mut snapshots = peers(10, 10); + let before = activity(9); + let mut after = activity(11); + match case { + "missing_peer" => { + snapshots.pop(); + } + "duplicate_peer" => { + snapshots[1] = snapshots[0].clone(); + } + "old_snapshot" => snapshots[0].1.generation = 8, + "future_snapshot" => snapshots[0].1.generation = 12, + "overflow" => after = activity(u64::MAX), + "owner" => snapshots[0].1.owner_id = uuid::Uuid::nil().to_string(), + "instance" => snapshots[0].1.instance_id = uuid::Uuid::from_u128(999).to_string(), + "protocol" => snapshots[0].1.protocol_version += 1, + "incomplete" => snapshots[0].1.complete = false, + "count" => snapshots[0].1.pending_bucket_count += 1, + "future_entry" => snapshots[0].1.buckets.get_mut("cold-bucket").expect("cold").generation = 11, + "nil_incarnation" => { + snapshots[0] + .1 + .buckets + .get_mut("cold-bucket") + .expect("cold") + .bucket_incarnation = uuid::Uuid::nil() + } + "empty_bucket" => { + let entry = snapshots[0].1.buckets.remove("cold-bucket").expect("cold"); + snapshots[0].1.buckets.insert(String::new(), entry); + } + "entry_limit" => { + let peer = &mut snapshots[0].1; + peer.buckets = (0..=crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES) + .map(|index| { + ( + format!("bucket-{index}"), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: COLD, + generation: 8, + }, + ) + }) + .collect(); + peer.pending_bucket_count = peer.buckets.len() as u64; + } + "pending" => { + snapshots[0].1.generation = 11; + snapshots[0].1.buckets.clear(); + snapshots[0].1.pending_bucket_count = 0; + } + _ => unreachable!(), + } + assert!(ScannerRemoteDirtyUsageWindow::verify(before, snapshots, after).is_none(), "{case}"); + } + } + + #[test] + fn certification_rejects_owner_replacement_and_a_generation_rollback_between_windows() { + let mut changed = window(12, 13, 14); + changed.peers.get_mut("peer-b").expect("clean peer").owner_id = uuid::Uuid::from_u128(999).to_string(); + assert!(ScannerRemoteDirtyUsageCertification::new(window(9, 10, 11), changed).is_none()); + assert!(ScannerRemoteDirtyUsageCertification::new(window(9, 10, 11), window(8, 9, 10)).is_none()); + } + + #[test] + fn certification_requires_actual_coverage_and_the_same_incarnation_on_every_set() { + let proof = ScannerRemoteDirtyUsageCertification::new(window(10, 10, 10), window(10, 10, 10)).expect("stable peers"); + for case in ["missing_set", "missing_bucket", "wrong_incarnation", "partial"] { + let (sources, mut completed) = completions(); + let mut uncertified = HashSet::new(); + match case { + "missing_set" => { + completed.remove(&DataUsageCacheSource::new(0, 1)); + } + "missing_bucket" => { + completed + .get_mut(&DataUsageCacheSource::new(0, 1)) + .expect("set") + .remove("cold-bucket"); + } + "wrong_incarnation" => { + completed + .get_mut(&DataUsageCacheSource::new(0, 1)) + .expect("set") + .insert("cold-bucket".to_string(), uuid::Uuid::from_u128(999)); + } + "partial" => { + uncertified.insert("cold-bucket".to_string()); + } + _ => unreachable!(), + } + let (acks, pending) = proof.classify_buckets(&sources, &completed, &uncertified); + assert!(pending.contains("cold-bucket"), "{case}"); + assert!(acks.iter().all(|ack| matches!(&ack.kind, ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } if entries.iter().all(|entry| entry.bucket != "cold-bucket"))), "{case}"); + } + } + + #[test] + fn a_bucket_changed_on_one_peer_cannot_be_acknowledged_on_another() { + let start = window(10, 10, 10); + let mut post_peers = peers(10, 10); + post_peers[1].1.generation = 8; + post_peers[1].1.pending_bucket_count = 1; + post_peers[1].1.buckets.insert( + "cold-bucket".to_string(), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: COLD, + generation: 8, + }, + ); + let mut after = activity(10); + after.insert( + "peer-b".to_string(), + crate::scanner::scanner_node_activity_for_tests(&post_peers[1].1.instance_id, 8, 8, true), + ); + let post = ScannerRemoteDirtyUsageWindow::verify(activity(10), post_peers, after).expect("clean peer became dirty"); + let proof = ScannerRemoteDirtyUsageCertification::new(start, post).expect("same producers"); + let (sources, completed) = completions(); + let (acks, pending) = proof.classify_buckets(&sources, &completed, &HashSet::new()); + assert!(pending.contains("cold-bucket")); + assert!(acks.iter().all(|ack| matches!(&ack.kind, ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } if entries.iter().all(|entry| entry.bucket != "cold-bucket")))); + } + + #[test] + fn start_window_trailing_activity_invalidates_same_cycle_current_cache_without_segment_reuse() { + let window = window(9, 10, 11); + let old_plan = scanner_bucket_cache_digest( + DataUsageScanPlanDigest(crate::scanner::scanner_activity_snapshot_digest(&window.before)), + None, + ); + let new_plan = scanner_bucket_cache_digest( + DataUsageScanPlanDigest(crate::scanner::scanner_activity_snapshot_digest(&window.activity_for_walk())), + None, + ); + let source = DataUsageCacheSource::new(0, 0); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "cold-bucket".to_string(), + next_cycle: 7, + leader_epoch: 11, + source: Some(source), + last_update: Some(SystemTime::UNIX_EPOCH), + snapshot_complete: true, + scan_plan_digest: Some(old_plan), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + ..Default::default() + }, + ..Default::default() + }; + cache.replace("cold-bucket", "", DataUsageEntry::default()); + let options = || DataUsageCacheReuseOptions { + require_source: true, + tier_registry_generation: None, + checkpoint_identity: None, + checkpoint_plan_digest: None, + }; + assert!(matches!( + current_cache_root_or_prepare_with_generation(&mut cache.clone(), "cold-bucket", source, 7, 11, old_plan, options()), + DataUsageCacheScanState::Current(_) + )); + assert!(matches!( + current_cache_root_or_prepare_with_generation(&mut cache, "cold-bucket", source, 7, 11, new_plan, options()), + DataUsageCacheScanState::Prepared { + outcome: DataUsageCachePrepareOutcome::Reset, + .. + } + )); + assert!(window.strict_remote_usage().is_none()); + } +} diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index c733a1371..a3fa8b4d7 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -1330,6 +1330,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() requires_full_scan, resolved_scope_observer: Some(observer), post_walk_gate: None, + peer_observations: None, service_cohort: None, }, ), @@ -1421,6 +1422,7 @@ async fn scoped_scan_same_cycle_maintenance_rewalks_after_root_delivery_failure( requires_full_scan: false, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, service_cohort: None, }, ), @@ -1475,6 +1477,7 @@ async fn scoped_scan_same_cycle_maintenance_rewalks_after_root_delivery_failure( requires_full_scan, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, service_cohort: None, }, ), @@ -2324,6 +2327,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers publication_epoch: Some(epoch), dirty_usage_buckets: Arc::new(HashMap::new()), bucket_failures: ScannerBucketFailureState::default(), + bucket_completions: None, pending_maintenance_work: Arc::new(AtomicBool::new(false)), cache_cycle_floor: Arc::new(AtomicU64::new(8)), cold_zero_walk_reuse_observed: Arc::new(AtomicBool::new(false)), @@ -3241,6 +3245,7 @@ async fn distributed_scoped_scan_falls_back_when_remote_ack_exceeds_protocol_bat store.as_ref(), true, super::io_cycle::ScannerBucketScopeResolution { + remote_window: None, requested_scope: ScannerBucketScanScope::default(), baseline_proof: ScannerCacheBaselineProof { authoritative_data: Some(&baseline), @@ -3300,6 +3305,7 @@ async fn distributed_scoped_scan_falls_back_when_remote_scoped_ack_capability_is store.as_ref(), true, super::io_cycle::ScannerBucketScopeResolution { + remote_window: None, requested_scope: ScannerBucketScanScope::default(), baseline_proof: ScannerCacheBaselineProof { authoritative_data: Some(&baseline), @@ -4868,6 +4874,7 @@ async fn replication_maintenance_refreshes_compacted_stats_across_cycles_and_ret service_cohort: None, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ) .await @@ -4975,6 +4982,7 @@ async fn replication_maintenance_refreshes_compacted_stats_across_cycles_and_ret service_cohort: None, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ) .await @@ -5122,6 +5130,7 @@ async fn run_full_cycle(store: &Arc, cycle: u64) -> ScannerCycleStatus resolved_scope_observer: None, service_cohort: None, post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/scanner_io/tests/checkpoint_coverage_identity.rs b/crates/scanner/src/scanner_io/tests/checkpoint_coverage_identity.rs index 72d80541e..e4f07a0ba 100644 --- a/crates/scanner/src/scanner_io/tests/checkpoint_coverage_identity.rs +++ b/crates/scanner/src/scanner_io/tests/checkpoint_coverage_identity.rs @@ -54,6 +54,7 @@ async fn run_full_cycle( service_cohort: None, resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/scanner_io/tests/collector_lifecycle.rs b/crates/scanner/src/scanner_io/tests/collector_lifecycle.rs index d0e85505f..c698a419d 100644 --- a/crates/scanner/src/scanner_io/tests/collector_lifecycle.rs +++ b/crates/scanner/src/scanner_io/tests/collector_lifecycle.rs @@ -140,6 +140,7 @@ async fn scan_plan(set: &Arc, names: &[&str], cycle: u64) -> ScannerBu ), dirty_usage_buckets: Arc::new(dirty_usage_buckets_for_tests()), bucket_failures: ScannerBucketFailureState::default(), + bucket_completions: None, pending_maintenance_work: Arc::new(AtomicBool::new(false)), cache_cycle_floor: Arc::new(AtomicU64::new(cycle)), cold_zero_walk_reuse_observed: Arc::new(AtomicBool::new(false)), diff --git a/crates/scanner/src/scanner_io/tests/listing_window_scope.rs b/crates/scanner/src/scanner_io/tests/listing_window_scope.rs index 4c4b40bbd..e56515aa8 100644 --- a/crates/scanner/src/scanner_io/tests/listing_window_scope.rs +++ b/crates/scanner/src/scanner_io/tests/listing_window_scope.rs @@ -47,6 +47,7 @@ async fn run_cycle( service_cohort: None, resolved_scope_observer: Some(observer), post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/scanner_io/tests/partial_bucket_carry.rs b/crates/scanner/src/scanner_io/tests/partial_bucket_carry.rs index bd1fec3ad..88bffbacb 100644 --- a/crates/scanner/src/scanner_io/tests/partial_bucket_carry.rs +++ b/crates/scanner/src/scanner_io/tests/partial_bucket_carry.rs @@ -57,6 +57,7 @@ async fn run_cycle(store: &Arc, cycle: u64, options: CycleOptions) -> ( service_cohort: None, resolved_scope_observer: options.resolved_scope_observer, post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/scanner_io/tests/per_bucket_certification.rs b/crates/scanner/src/scanner_io/tests/per_bucket_certification.rs index c3eca6776..17bb008bb 100644 --- a/crates/scanner/src/scanner_io/tests/per_bucket_certification.rs +++ b/crates/scanner/src/scanner_io/tests/per_bucket_certification.rs @@ -22,6 +22,7 @@ async fn put_fixture_object(store: &Arc, bucket: &str, object: &str) { struct CycleHooks { resolved_scope_observer: Option>, post_walk_gate: Option>, + peer_observations: Option, } async fn run_cycle( @@ -31,6 +32,7 @@ async fn run_cycle( requires_full_scan: bool, hooks: CycleHooks, ) -> (ScannerCycleResult, DataUsageInfo) { + let peer_cycle = hooks.peer_observations.is_some(); let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); let (updates, mut receiver) = mpsc::channel(1); @@ -52,12 +54,20 @@ async fn run_cycle( service_cohort: None, resolved_scope_observer: hooks.resolved_scope_observer, post_walk_gate: hooks.post_walk_gate, + peer_observations: hooks.peer_observations, }, ), ) .await .expect("cycle should finish within the test deadline") .expect("cycle should succeed"); + if peer_cycle { + assert_eq!( + result.status, + ScannerCycleStatus::Complete, + "one remote hot bucket cannot supersede certified cold buckets" + ); + } let snapshot = receiver.recv().await.expect("cycle should publish a snapshot"); assert!(receiver.recv().await.is_none(), "a cycle publishes one terminal snapshot"); (result, snapshot) @@ -91,6 +101,7 @@ async fn write_during_walk_leaves_only_that_bucket_pending_and_scopes_the_next_c CycleHooks { resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ) .await; @@ -128,6 +139,7 @@ async fn write_during_walk_leaves_only_that_bucket_pending_and_scopes_the_next_c CycleHooks { resolved_scope_observer: Some(observer), post_walk_gate: Some(gate), + peer_observations: None, }, ) .await; @@ -175,6 +187,7 @@ async fn write_during_walk_leaves_only_that_bucket_pending_and_scopes_the_next_c CycleHooks { resolved_scope_observer: Some(observer), post_walk_gate: None, + peer_observations: None, }, ) .await; @@ -226,6 +239,7 @@ async fn pending_put_tail_across_the_walk_no_longer_vetoes_the_cycle() { CycleHooks { resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ) .await; @@ -285,6 +299,7 @@ async fn guardrail_veto_still_supersedes_the_cycle_onto_the_observed_route() { CycleHooks { resolved_scope_observer: Some(observer), post_walk_gate: Some(gate), + peer_observations: None, }, ) .await; @@ -306,3 +321,230 @@ async fn guardrail_veto_still_supersedes_the_cycle_onto_the_observed_route() { ); clear_dirty_usage_buckets_for_tests(); } + +#[tokio::test] +#[serial] +async fn remote_hot_bucket_keeps_cold_buckets_certified_and_scopes_the_next_cycle() { + run_remote_certification_cycle(false).await; +} + +#[tokio::test] +#[serial] +async fn a_verified_empty_ack_on_a_full_scope_keeps_the_remote_hot_bucket_pending() { + run_remote_certification_cycle(true).await; +} + +async fn run_remote_certification_cycle(empty_ack: bool) { + let (_temp_dir, store) = setup_local_scanner_store(1, 1).await; + clear_dirty_usage_buckets_for_tests(); + let mut incarnations = BTreeMap::new(); + for bucket in BUCKETS { + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("fixture bucket"); + put_fixture_object(&store, bucket, "initial").await; + record_dirty_usage_bucket(bucket); + incarnations.insert( + bucket.to_string(), + store.all_set_disks()[0] + .find_bucket_incarnation_id_from_disk(bucket) + .await + .expect("read actual incarnation") + .expect("fixture bucket has an incarnation"), + ); + } + wait_for_namespace_commit_tails(store.as_ref()).await; + let peers = Arc::new(StdMutex::new(BTreeMap::from([ + ( + "peer-a:9000".to_string(), + peer_dirty_usage_snapshot(&Uuid::from_u128(101).to_string(), 7, true, &[]), + ), + ( + "peer-b:9000".to_string(), + peer_dirty_usage_snapshot(&Uuid::from_u128(102).to_string(), 7, true, &[]), + ), + ]))); + // These are the RPC observations only. Bucket listing, traversal, cache + // coverage, classification and candidate production use the real cycle. + let activity_peers = Arc::clone(&peers); + let dirty_peers = Arc::clone(&peers); + let observations = super::super::io_cycle::ScannerCyclePeerObservations { + activity: Arc::new(move || { + activity_peers + .lock() + .expect("peer fixture lock") + .iter() + .map(|(host, peer)| { + ( + host.clone(), + crate::scanner::scanner_node_activity_for_tests( + &peer.instance_id, + peer.generation, + peer.generation, + !peer.buckets.is_empty(), + ), + ) + }) + .collect() + }), + dirty_usage: Arc::new(move || { + dirty_peers + .lock() + .expect("peer fixture lock") + .iter() + .map(|(host, snapshot)| (host.clone(), snapshot.clone())) + .collect() + }), + scoped_ack_capable: true, + }; + let (result, baseline) = run_cycle( + &store, + 1, + None, + true, + CycleHooks { + resolved_scope_observer: None, + post_walk_gate: None, + peer_observations: Some(observations.clone()), + }, + ) + .await; + assert_eq!(result.status, ScannerCycleStatus::Complete); + assert!(result.clear_verified_usage().is_empty()); + assert!(baseline.is_complete_bucket_usage_snapshot()); + let initial_dirty = if empty_ack { + vec!["hot-bucket"] + } else { + vec!["cold-bucket", "idle-bucket"] + }; + for bucket in initial_dirty { + put_fixture_object(&store, bucket, "second").await; + let mut locked = peers.lock().expect("peer fixture lock"); + let peer = locked.get_mut("peer-a:9000").expect("remote producer exists"); + peer.generation += 1; + peer.buckets.insert( + bucket.to_string(), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: incarnations[bucket], + generation: peer.generation, + }, + ); + peer.pending_bucket_count = u64::try_from(peer.buckets.len()).expect("bounded fixture"); + } + wait_for_namespace_commit_tails(store.as_ref()).await; + let (observer, resolved_scope) = tokio::sync::oneshot::channel(); + let (release, gate) = tokio::sync::oneshot::channel(); + let writer_store = Arc::clone(&store); + let writer_peers = Arc::clone(&peers); + let hot_incarnation = incarnations["hot-bucket"]; + let writer = tokio::spawn(async move { + let scope = resolved_scope.await.expect("real cycle resolved its scope"); + put_fixture_object(&writer_store, "hot-bucket", "during-walk").await; + { + let mut locked = writer_peers.lock().expect("peer fixture lock"); + let peer = locked.get_mut("peer-a:9000").expect("remote producer exists"); + peer.generation += 1; + peer.buckets.insert( + "hot-bucket".to_string(), + crate::storage_api::EcstoreScannerPeerDirtyUsageBucket { + bucket_incarnation: hot_incarnation, + generation: peer.generation, + }, + ); + peer.pending_bucket_count = u64::try_from(peer.buckets.len()).expect("bounded fixture"); + } + wait_for_namespace_commit_tails(writer_store.as_ref()).await; + release.send(()).expect("cycle awaits its actual post-walk proof"); + scope + }); + let (result, published) = run_cycle( + &store, + 2, + (!empty_ack).then(|| baseline_bytes(&baseline)), + false, + CycleHooks { + resolved_scope_observer: Some(observer), + post_walk_gate: Some(gate), + peer_observations: Some(observations.clone()), + }, + ) + .await; + let scope = writer.await.expect("writer completed"); + if empty_ack { + assert!(scope.is_default(), "this case exercises the full-scope generation fallback"); + } else { + assert_eq!( + scope.selected_buckets_for_tests(), + Some(&HashSet::from(["cold-bucket".to_string(), "idle-bucket".to_string()])) + ); + } + assert_eq!( + result.status, + ScannerCycleStatus::Complete, + "one remote hot bucket cannot supersede certified cold buckets" + ); + assert_eq!(published.usage_snapshot_pending_buckets, vec!["hot-bucket".to_string()]); + assert_eq!(published.usage_snapshot_converged, Some(false)); + assert!(!published.usage_snapshot_observational); + assert_eq!(published.buckets_usage["cold-bucket"].objects_count, if empty_ack { 1 } else { 2 }); + assert_eq!(published.buckets_usage["idle-bucket"].objects_count, if empty_ack { 1 } else { 2 }); + if !empty_ack { + assert_eq!(published.buckets_usage["hot-bucket"].objects_count, 1); + } + assert!(result.has_remote_dirty_usage_pending()); + let acknowledgements = &result.remote_dirty_usage_acknowledgements; + if empty_ack { + assert!(acknowledgements.is_empty(), "a verified empty scoped ACK cannot become a generation ACK"); + } else { + assert_eq!(acknowledgements.len(), 1, "the clean peer is proved without a fabricated ACK"); + let crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { owner_id, entries } = &acknowledgements[0].kind else { + panic!("certified remote buckets require exact scoped ACKs"); + }; + assert_eq!( + entries.iter().map(|entry| entry.bucket.as_str()).collect::>(), + BTreeSet::from(["cold-bucket", "idle-bucket"]) + ); + { + let mut locked = peers.lock().expect("peer fixture lock"); + let peer = locked + .get_mut(&acknowledgements[0].host) + .expect("ACK targets the original peer"); + assert_eq!(&peer.owner_id, owner_id); + assert_eq!(peer.instance_id, acknowledgements[0].instance_id); + // Model the RPC receiver only after asserting the production sender's + // exact scope; this does not claim runtime publication/receiver proof. + for entry in entries { + let current = peer.buckets.get(&entry.bucket).expect("ACK names an observed tuple"); + assert_eq!( + (current.bucket_incarnation, current.generation), + (entry.bucket_incarnation, entry.generation) + ); + peer.buckets.remove(&entry.bucket); + } + peer.pending_bucket_count = u64::try_from(peer.buckets.len()).expect("bounded fixture"); + } + } + let _ = result.clear_verified_usage(); + let (observer, resolved_scope) = tokio::sync::oneshot::channel(); + let (result, next) = run_cycle( + &store, + 3, + Some(baseline_bytes(&published)), + false, + CycleHooks { + resolved_scope_observer: Some(observer), + post_walk_gate: None, + peer_observations: Some(observations), + }, + ) + .await; + assert_eq!( + resolved_scope.await.expect("next cycle scope").selected_buckets_for_tests(), + Some(&HashSet::from(["hot-bucket".to_string()])) + ); + assert_eq!(result.status, ScannerCycleStatus::Complete); + assert_eq!(next.buckets_usage["hot-bucket"].objects_count, if empty_ack { 3 } else { 2 }); + assert_eq!(next.usage_snapshot_converged, Some(true)); + clear_dirty_usage_buckets_for_tests(); +} diff --git a/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs index f31ae0f0d..a7584182f 100644 --- a/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs +++ b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs @@ -139,6 +139,7 @@ async fn run_entry( service_cohort: None, resolved_scope_observer: Some(observer), post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/scanner_io/tests/service_cohort.rs b/crates/scanner/src/scanner_io/tests/service_cohort.rs index 1deb5d6d1..e53447b33 100644 --- a/crates/scanner/src/scanner_io/tests/service_cohort.rs +++ b/crates/scanner/src/scanner_io/tests/service_cohort.rs @@ -74,6 +74,7 @@ async fn run_cohort_cycle( service_cohort: Some(cohort), resolved_scope_observer: None, post_walk_gate: None, + peer_observations: None, }, ), ) diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 349f76981..b94cc5596 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -108,8 +108,10 @@ pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, SetupType as EcstoreSetupType, }; -pub(crate) use rustfs_ecstore::api::notification::NotificationSys as EcstoreNotificationSys; pub(crate) use rustfs_ecstore::api::notification::scanner_peer_transport_error_message_is_retryable; +pub(crate) use rustfs_ecstore::api::notification::{ + NotificationSys as EcstoreNotificationSys, ScannerPublicationLeaseGrant as EcstoreScannerPublicationLeaseGrant, +}; pub(crate) use rustfs_ecstore::api::object::{ SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitState, }; @@ -118,8 +120,6 @@ pub(crate) use rustfs_ecstore::api::rebalance::{ RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta, RebalanceStats as EcstoreRebalanceStats, }; -#[cfg(test)] -pub(crate) use rustfs_ecstore::api::rpc::ScannerPeerDirtyUsageBucket as EcstoreScannerPeerDirtyUsageBucket; pub(crate) use rustfs_ecstore::api::rpc::{ ScannerBucketListing as EcstoreScannerBucketListing, ScannerDirtyUsageAcknowledgement as EcstoreScannerDirtyUsageAcknowledgement, @@ -127,6 +127,10 @@ pub(crate) use rustfs_ecstore::api::rpc::{ ScannerScopedDirtyUsageAckEntry as EcstoreScannerScopedDirtyUsageAckEntry, }; #[cfg(test)] +pub(crate) use rustfs_ecstore::api::rpc::{ + ScannerPeerDirtyUsageBucket as EcstoreScannerPeerDirtyUsageBucket, ScannerPublicationLease as EcstoreScannerPublicationLease, +}; +#[cfg(test)] pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext; #[cfg(test)] pub(crate) use rustfs_ecstore::api::runtime::set_object_store_resolver as ecstore_set_object_store_resolver; diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7338ed8b3..8d6e375a7 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -68,7 +68,7 @@ license = [] io-scheduler-debug = [] # Enable debug information in I/O scheduler tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only) full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"] -e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks"] +e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"] # Shortens Connect credentials only in debug E2E builds. connect-e2e-short-credentials = [] # Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root. diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 59280a6ee..e38c55a2e 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1396,8 +1396,13 @@ impl scanner_control_service_server::ScannerControlService for NodeService { .zip(&request.entries) .map(|(guard, entry)| (guard, entry.generation)) .collect::>(); - rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only) - .map_err(|err| Status::failed_precondition(err.to_string())) + #[cfg(feature = "e2e-test-hooks")] + observe_scoped_dirty_usage_ack("ack_before_apply", &request, None).await?; + let cleared = rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only) + .map_err(|err| Status::failed_precondition(err.to_string()))?; + #[cfg(feature = "e2e-test-hooks")] + observe_scoped_dirty_usage_ack("ack_after_apply", &request, Some(cleared)).await?; + Ok::<_, Status>(cleared) }) .await .map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??; @@ -1420,6 +1425,44 @@ impl scanner_control_service_server::ScannerControlService for NodeService { } } +/// Capture the real compare-and-remove boundary while the production +/// incarnation fences are still held. Capability probes never pause here. +#[cfg(feature = "e2e-test-hooks")] +async fn observe_scoped_dirty_usage_ack( + stage: &str, + request: &ScannerScopedDirtyUsageAckRequest, + cleared: Option, +) -> Result<(), Status> { + if request.probe_only { + return Ok(()); + } + let dirty = + rustfs_scanner::scanner_dirty_usage_snapshot(rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize); + let detail = serde_json::json!({ + "owner_id": request.owner_id, + "instance_id": request.instance_id, + "probe_only": request.probe_only, + "entries": request.entries.iter().map(|entry| serde_json::json!({ + "bucket": entry.bucket, + "bucket_incarnation": Uuid::from_slice(entry.bucket_incarnation.as_ref()).ok().map(|value| value.to_string()), + "generation": entry.generation, + })).collect::>(), + "cleared": cleared, + "dirty_snapshot": { + "generation": dirty.generation, + "pending_bucket_count": dirty.pending_bucket_count, + "complete": dirty.complete, + "buckets": dirty.buckets.iter().map(|bucket| serde_json::json!({ + "bucket": bucket.bucket, + "generation": bucket.generation, + })).collect::>(), + }, + }); + rustfs_scanner::e2e_barrier::checkpoint(stage, detail) + .await + .map_err(|error| Status::internal(format!("scoped dirty usage test barrier failed: {error}"))) +} + #[tonic::async_trait] impl Node for NodeService { async fn ping(&self, request: Request) -> Result, Status> {